From 1c5a5f21a339523ad5984287d9194cb511659136 Mon Sep 17 00:00:00 2001 From: Matt Kwong <mattkwong@google.com> Date: Tue, 12 Sep 2017 17:44:37 -0700 Subject: [PATCH 01/20] Add .NET CLI to coverage Dockerfile --- .../multilang_jessie_x64/Dockerfile.template | 1 + .../test/multilang_jessie_x64/Dockerfile | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/templates/tools/dockerfile/test/multilang_jessie_x64/Dockerfile.template b/templates/tools/dockerfile/test/multilang_jessie_x64/Dockerfile.template index 6cad474a206..0d47aa91f36 100644 --- a/templates/tools/dockerfile/test/multilang_jessie_x64/Dockerfile.template +++ b/templates/tools/dockerfile/test/multilang_jessie_x64/Dockerfile.template @@ -19,6 +19,7 @@ <%include file="../../apt_get_basic.include"/> <%include file="../../gcp_api_libraries.include"/> <%include file="../../csharp_deps.include"/> + <%include file="../../csharp_dotnetcli_deps.include"/> <%include file="../../cxx_deps.include"/> <%include file="../../node_deps.include"/> <%include file="../../php_deps.include"/> diff --git a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile index b7373b5d9c3..6f1759bceb3 100644 --- a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile @@ -70,6 +70,24 @@ RUN apt-get update && apt-get -y dist-upgrade && apt-get install -y \ RUN nuget update -self +# Install dotnet SDK based on https://www.microsoft.com/net/core#debian +RUN apt-get update && apt-get install -y curl libunwind8 gettext +# dotnet-dev-1.0.0-preview2-003131 +RUN curl -sSL -o dotnet100.tar.gz https://go.microsoft.com/fwlink/?LinkID=827530 +RUN mkdir -p /opt/dotnet && tar zxf dotnet100.tar.gz -C /opt/dotnet +# dotnet-dev-1.0.1 +RUN curl -sSL -o dotnet101.tar.gz https://go.microsoft.com/fwlink/?LinkID=843453 +RUN mkdir -p /opt/dotnet && tar zxf dotnet101.tar.gz -C /opt/dotnet +RUN ln -s /opt/dotnet/dotnet /usr/local/bin + +# Trigger the population of the local package cache +ENV NUGET_XMLDOC_MODE skip +RUN mkdir warmup \ + && cd warmup \ + && dotnet new \ + && cd .. \ + && rm -rf warmup + #================= # C++ dependencies RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang && apt-get clean From 0c867bed5bb0425c57dd539e73c790ae3995d864 Mon Sep 17 00:00:00 2001 From: Matt Kwong <mattkwong@google.com> Date: Thu, 21 Sep 2017 16:25:48 -0700 Subject: [PATCH 02/20] Add -lstdc++ flag for compiling under gcov --- Makefile | 2 +- binding.gyp | 1 + build.yaml | 2 +- grpc.gyp | 1 + 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 0c3648566a9..7e65718485f 100644 --- a/Makefile +++ b/Makefile @@ -187,7 +187,7 @@ CXX_gcov = g++ LD_gcov = gcc LDXX_gcov = g++ CPPFLAGS_gcov = -O0 -fprofile-arcs -ftest-coverage -Wno-return-type -LDFLAGS_gcov = -fprofile-arcs -ftest-coverage -rdynamic +LDFLAGS_gcov = -fprofile-arcs -ftest-coverage -rdynamic -lstdc++ DEFINES_gcov = _DEBUG DEBUG GPR_GCOV VALID_CONFIG_memcheck = 1 diff --git a/binding.gyp b/binding.gyp index 06dc731935c..3d07dda6d63 100644 --- a/binding.gyp +++ b/binding.gyp @@ -101,6 +101,7 @@ '-fprofile-arcs', '-ftest-coverage', '-rdynamic', + '-lstdc++', ], }], ['grpc_alpine=="true"', { diff --git a/build.yaml b/build.yaml index d15cf0339ca..635c908e099 100644 --- a/build.yaml +++ b/build.yaml @@ -4778,7 +4778,7 @@ configs: CXX: g++ DEFINES: _DEBUG DEBUG GPR_GCOV LD: gcc - LDFLAGS: -fprofile-arcs -ftest-coverage -rdynamic + LDFLAGS: -fprofile-arcs -ftest-coverage -rdynamic -lstdc++ LDXX: g++ helgrind: CPPFLAGS: -O0 diff --git a/grpc.gyp b/grpc.gyp index 48fb2ba7cf2..6be7050fcab 100644 --- a/grpc.gyp +++ b/grpc.gyp @@ -98,6 +98,7 @@ '-fprofile-arcs', '-ftest-coverage', '-rdynamic', + '-lstdc++', ], }], ['grpc_alpine=="true"', { From c111341a178e526e3b92573514db0910acc69ccb Mon Sep 17 00:00:00 2001 From: Matt Kwong <mattkwong@google.com> Date: Fri, 22 Sep 2017 11:25:34 -0700 Subject: [PATCH 03/20] Make Python generate coverage report in gcov mode --- tools/run_tests/run_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index b66c5f7f71f..86fd7b34b21 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -639,7 +639,7 @@ class PythonLanguage(object): return [config.build for config in self.pythons] def post_tests_steps(self): - if self.config != 'gcov': + if self.config.build_config != 'gcov': return [] else: return [['tools/run_tests/helper_scripts/post_tests_python.sh']] From c8e145bd93d3aa0ae02fe8bbddc265f1c35b5339 Mon Sep 17 00:00:00 2001 From: ZhouyihaiDing <ddyihai@google.com> Date: Mon, 25 Sep 2017 23:20:30 +0000 Subject: [PATCH 04/20] add protobuf c_ext to php benchmark --- src/ruby/qps/proxy-worker.rb | 16 +++++++++-- tools/jenkins/run_full_performance.sh | 2 +- tools/run_tests/performance/run_worker_php.sh | 11 ++++++-- .../run_tests/performance/scenario_config.py | 28 +++++++++++++++++++ 4 files changed, 50 insertions(+), 7 deletions(-) diff --git a/src/ruby/qps/proxy-worker.rb b/src/ruby/qps/proxy-worker.rb index 488610ae74d..ae7006e7d60 100755 --- a/src/ruby/qps/proxy-worker.rb +++ b/src/ruby/qps/proxy-worker.rb @@ -31,8 +31,9 @@ require 'src/proto/grpc/testing/services_services_pb' require 'src/proto/grpc/testing/proxy-service_services_pb' class ProxyBenchmarkClientServiceImpl < Grpc::Testing::ProxyClientService::Service - def initialize(port) + def initialize(port, c_ext) @mytarget = "localhost:" + port.to_s + @use_c_ext = c_ext end def setup(config) @config = config @@ -41,7 +42,13 @@ class ProxyBenchmarkClientServiceImpl < Grpc::Testing::ProxyClientService::Servi @histogram = Histogram.new(@histres, @histmax) @start_time = Time.now # TODO(vjpai): Support multiple client channels by spawning off a PHP client per channel - command = "php -d extension=" + File.expand_path(File.dirname(__FILE__)) + "/../../php/ext/grpc/modules/grpc.so " + File.expand_path(File.dirname(__FILE__)) + "/../../php/tests/qps/client.php " + @mytarget + if @use_c_ext + puts "Use protobuf c extension" + command = "php -d extension=" + File.expand_path(File.dirname(__FILE__)) + "/../../php/tests/qps/vendor/google/protobuf/php/ext/google/protobuf/modules/protobuf.so " + "-d extension=" + File.expand_path(File.dirname(__FILE__)) + "/../../php/ext/grpc/modules/grpc.so " + File.expand_path(File.dirname(__FILE__)) + "/../../php/tests/qps/client.php " + @mytarget + else + puts "Use protobuf php extension" + command = "php -d extension=" + File.expand_path(File.dirname(__FILE__)) + "/../../php/ext/grpc/modules/grpc.so " + File.expand_path(File.dirname(__FILE__)) + "/../../php/tests/qps/client.php " + @mytarget + end puts "Starting command: " + command @php_pid = spawn(command) end @@ -128,6 +135,9 @@ def proxymain opts.on('--driver_port PORT', '<port>') do |v| options['driver_port'] = v end + opts.on("-c", "--[no-]c_proto_ext", "Use protobuf C-extention") do |c| + options[:c_ext] = c + end end.parse! # Configure any errors with client or server child threads to surface @@ -136,7 +146,7 @@ def proxymain s = GRPC::RpcServer.new port = s.add_http2_port("0.0.0.0:" + options['driver_port'].to_s, :this_port_is_insecure) - bmc = ProxyBenchmarkClientServiceImpl.new(port) + bmc = ProxyBenchmarkClientServiceImpl.new(port, options[:c_ext]) s.handle(bmc) s.handle(ProxyWorkerServiceImpl.new(s, bmc)) s.run diff --git a/tools/jenkins/run_full_performance.sh b/tools/jenkins/run_full_performance.sh index a9661c7e266..aae76b0fde8 100755 --- a/tools/jenkins/run_full_performance.sh +++ b/tools/jenkins/run_full_performance.sh @@ -21,7 +21,7 @@ cd $(dirname $0)/../.. # run 8core client vs 8core server tools/run_tests/run_performance_tests.py \ - -l c++ csharp node ruby java python go node_express php \ + -l c++ csharp node ruby java python go node_express php php_ext \ --netperf \ --category scalable \ --bq_result_table performance_test.performance_experiment \ diff --git a/tools/run_tests/performance/run_worker_php.sh b/tools/run_tests/performance/run_worker_php.sh index 5d0c4fa4fd3..e524d5286d0 100755 --- a/tools/run_tests/performance/run_worker_php.sh +++ b/tools/run_tests/performance/run_worker_php.sh @@ -16,13 +16,18 @@ source ~/.rvm/scripts/rvm set -ex -repo=$(dirname $0)/../../.. - +cd $(dirname $0)/../../.. +repo=$(pwd) # First set up all dependences needed for PHP QPS test cd $repo cd src/php/tests/qps composer install +# Install protobuf C-extension for php +cd vendor/google/protobuf/php/ext/google/protobuf +phpize +./configure +make # The proxy worker for PHP is implemented in Ruby -cd ../../../.. +cd $repo ruby src/ruby/qps/proxy-worker.rb $@ diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index 5efc9f56485..31017f1d85a 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -827,6 +827,33 @@ class PhpLanguage: return 'php' +class PhpLanguage_ext: + + def __init__(self): + pass + self.safename = str(self) + + def worker_cmdline(self): + return ['tools/run_tests/performance/run_worker_php.sh -c'] + + def worker_port_offset(self): + return 800 + + def scenarios(self): + yield _ping_pong_scenario( + 'php_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + server_language='c++', async_server_threads=1) + + yield _ping_pong_scenario( + 'php_to_cpp_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + server_language='c++', async_server_threads=1) + + def __str__(self): + return 'php' + + class JavaLanguage: def __init__(self): @@ -1025,6 +1052,7 @@ LANGUAGES = { 'node_express': NodeExpressLanguage(), 'ruby' : RubyLanguage(), 'php' : PhpLanguage(), + 'php_ext' : PhpLanguage_ext(), 'java' : JavaLanguage(), 'python' : PythonLanguage(), 'go' : GoLanguage(), From 5fef3e3fd56e0b978cf1df99d326251601b2a42b Mon Sep 17 00:00:00 2001 From: ZhouyihaiDing <ddyihai@google.com> Date: Tue, 26 Sep 2017 18:25:28 +0000 Subject: [PATCH 05/20] reuse class; change language name; scenario naame; --- tools/jenkins/run_full_performance.sh | 2 +- .../run_tests/performance/scenario_config.py | 48 ++++++------------- 2 files changed, 15 insertions(+), 35 deletions(-) diff --git a/tools/jenkins/run_full_performance.sh b/tools/jenkins/run_full_performance.sh index aae76b0fde8..9598fd77349 100755 --- a/tools/jenkins/run_full_performance.sh +++ b/tools/jenkins/run_full_performance.sh @@ -21,7 +21,7 @@ cd $(dirname $0)/../.. # run 8core client vs 8core server tools/run_tests/run_performance_tests.py \ - -l c++ csharp node ruby java python go node_express php php_ext \ + -l c++ csharp node ruby java python go node_express php_protobuf_php php_protobuf_c \ --netperf \ --category scalable \ --bq_result_table performance_test.performance_experiment \ diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index 31017f1d85a..5019358ab3e 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -802,52 +802,32 @@ class RubyLanguage: class PhpLanguage: - def __init__(self): + def __init__(self, use_protobuf_c_extension=False): pass + self.use_protobuf_c_extension=use_protobuf_c_extension self.safename = str(self) def worker_cmdline(self): + if self.use_protobuf_c_extension: + return ['tools/run_tests/performance/run_worker_php.sh -c'] return ['tools/run_tests/performance/run_worker_php.sh'] def worker_port_offset(self): return 800 def scenarios(self): + php_extension_mode='php_protobuf_php_extension' + if self.use_protobuf_c_extension: + php_extension_mode='php_protobuf_c_extension' + yield _ping_pong_scenario( - 'php_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', - client_type='SYNC_CLIENT', server_type='SYNC_SERVER', - server_language='c++', async_server_threads=1) - - yield _ping_pong_scenario( - 'php_to_cpp_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', - client_type='SYNC_CLIENT', server_type='SYNC_SERVER', - server_language='c++', async_server_threads=1) - - def __str__(self): - return 'php' - - -class PhpLanguage_ext: - - def __init__(self): - pass - self.safename = str(self) - - def worker_cmdline(self): - return ['tools/run_tests/performance/run_worker_php.sh -c'] - - def worker_port_offset(self): - return 800 - - def scenarios(self): - yield _ping_pong_scenario( - 'php_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', - client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + '%s_to_cpp_protobuf_sync_unary_ping_pong' % php_extension_mode, + rpc_type='UNARY', client_type='SYNC_CLIENT', server_type='SYNC_SERVER', server_language='c++', async_server_threads=1) yield _ping_pong_scenario( - 'php_to_cpp_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', - client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + '%s_to_cpp_protobuf_sync_streaming_ping_pong' % php_extension_mode, + rpc_type='STREAMING', client_type='SYNC_CLIENT', server_type='SYNC_SERVER', server_language='c++', async_server_threads=1) def __str__(self): @@ -1051,8 +1031,8 @@ LANGUAGES = { 'node' : NodeLanguage(), 'node_express': NodeExpressLanguage(), 'ruby' : RubyLanguage(), - 'php' : PhpLanguage(), - 'php_ext' : PhpLanguage_ext(), + 'php_protobuf_php' : PhpLanguage(), + 'php_protobuf_c' : PhpLanguage(use_protobuf_c_extension=True), 'java' : JavaLanguage(), 'python' : PythonLanguage(), 'go' : GoLanguage(), From 3069fc5eb8f8cde9187fad84d84ed507a769dc47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Vo=C3=9F?= <thomas.voss.bochum@gmail.com> Date: Wed, 27 Sep 2017 09:43:14 +0200 Subject: [PATCH 06/20] Increase reference count on state used in tcp connect. The state is used both in the callback for the actual connect as well as in the additional timeout that is setup for the operation. Both code paths decrease the reference count and if they happen to be queued at the same time, memory is corrupted. Subsequent behavior is undefined and segfaults can be observed as a result. Fixes #12608 --- src/core/lib/iomgr/tcp_client_uv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/tcp_client_uv.c b/src/core/lib/iomgr/tcp_client_uv.c index 786c456b735..f2b23aae2e6 100644 --- a/src/core/lib/iomgr/tcp_client_uv.c +++ b/src/core/lib/iomgr/tcp_client_uv.c @@ -145,7 +145,7 @@ static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx, connect->resource_quota = resource_quota; uv_tcp_init(uv_default_loop(), connect->tcp_handle); connect->connect_req.data = connect; - connect->refs = 1; + connect->refs = 2; // One for the connect operation, one for the timer. if (GRPC_TRACER_ON(grpc_tcp_trace)) { gpr_log(GPR_DEBUG, "CLIENT_CONNECT: %s: asynchronously connecting", From 316855ef9857636f5501678a7feac7b9cf9441fc Mon Sep 17 00:00:00 2001 From: Matt Kwong <mattkwong@google.com> Date: Wed, 27 Sep 2017 14:26:41 -0700 Subject: [PATCH 07/20] Don't stop on failure in post test steps --- tools/run_tests/run_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 86fd7b34b21..aafd8bba35a 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -1575,7 +1575,7 @@ def _build_and_run( suite_name=args.report_suite_name) number_failures, _ = jobset.run( - post_tests_steps, maxjobs=1, stop_on_failure=True, + post_tests_steps, maxjobs=1, stop_on_failure=False, newline_on_success=newline_on_success, travis=args.travis) out = [] From fa90655822ed6e02502f9d05b0479e12829042b9 Mon Sep 17 00:00:00 2001 From: Matt Kwong <mattkwong@google.com> Date: Thu, 28 Sep 2017 10:24:22 -0700 Subject: [PATCH 08/20] Add coverage test to Kokoro --- tools/internal_ci/linux/grpc_coverage.cfg | 24 ++++++++++++++++++ tools/internal_ci/linux/grpc_coverage.sh | 30 +++++++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 tools/internal_ci/linux/grpc_coverage.cfg create mode 100755 tools/internal_ci/linux/grpc_coverage.sh diff --git a/tools/internal_ci/linux/grpc_coverage.cfg b/tools/internal_ci/linux/grpc_coverage.cfg new file mode 100644 index 00000000000..56b7745b21f --- /dev/null +++ b/tools/internal_ci/linux/grpc_coverage.cfg @@ -0,0 +1,24 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_coverage.sh" +timeout_mins: 420 +action { + define_artifacts { + regex: "github/grpc/reports/**" + } +} diff --git a/tools/internal_ci/linux/grpc_coverage.sh b/tools/internal_ci/linux/grpc_coverage.sh new file mode 100755 index 00000000000..31623d2618c --- /dev/null +++ b/tools/internal_ci/linux/grpc_coverage.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# Enter the gRPC repo root +cd $(dirname $0)/../../.. + +source tools/internal_ci/helper_scripts/prepare_build_linux_rc + +python tools/run_tests/run_tests.py \ + --use_docker \ + -t \ + -l all \ + -c gcov \ + -x report.xml \ + -j 16 + \ No newline at end of file From bdf998a1840a5b35c9ad84d27c8ca4ce2642b06a Mon Sep 17 00:00:00 2001 From: Yash Tibrewal <yashkt@google.com> Date: Wed, 27 Sep 2017 17:07:43 -0700 Subject: [PATCH 09/20] Fix compression tests for when they expect the client to receive the completion tag --- test/core/end2end/tests/compressed_payload.c | 2 +- test/core/end2end/tests/stream_compression_compressed_payload.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/core/end2end/tests/compressed_payload.c b/test/core/end2end/tests/compressed_payload.c index ce86e97b83e..ba03773036b 100644 --- a/test/core/end2end/tests/compressed_payload.c +++ b/test/core/end2end/tests/compressed_payload.c @@ -193,6 +193,7 @@ static void request_for_disabled_algorithm( GPR_ASSERT(GRPC_CALL_OK == error); CQ_EXPECT_COMPLETION(cqv, tag(101), true); + CQ_EXPECT_COMPLETION(cqv, tag(1), true); cq_verify(cqv); op = ops; @@ -221,7 +222,6 @@ static void request_for_disabled_algorithm( GPR_ASSERT(GRPC_CALL_OK == error); CQ_EXPECT_COMPLETION(cqv, tag(103), true); - CQ_EXPECT_COMPLETION(cqv, tag(1), true); cq_verify(cqv); /* call was cancelled (closed) ... */ diff --git a/test/core/end2end/tests/stream_compression_compressed_payload.c b/test/core/end2end/tests/stream_compression_compressed_payload.c index 094c4de1c05..8b47741cd40 100644 --- a/test/core/end2end/tests/stream_compression_compressed_payload.c +++ b/test/core/end2end/tests/stream_compression_compressed_payload.c @@ -193,6 +193,7 @@ static void request_for_disabled_algorithm( GPR_ASSERT(GRPC_CALL_OK == error); CQ_EXPECT_COMPLETION(cqv, tag(101), true); + CQ_EXPECT_COMPLETION(cqv, tag(1), true); cq_verify(cqv); op = ops; @@ -221,7 +222,6 @@ static void request_for_disabled_algorithm( GPR_ASSERT(GRPC_CALL_OK == error); CQ_EXPECT_COMPLETION(cqv, tag(103), true); - CQ_EXPECT_COMPLETION(cqv, tag(1), true); cq_verify(cqv); /* call was cancelled (closed) ... */ From b5eaf7734cd8c85263afa5a773ce0f7a33d75b0a Mon Sep 17 00:00:00 2001 From: Yuchen Zeng <zyc@google.com> Date: Wed, 1 Feb 2017 22:15:02 -0800 Subject: [PATCH 10/20] gRPC CLI batch mode --- test/cpp/util/grpc_tool.cc | 125 ++++++++++++++++++++++++++++++-- test/cpp/util/grpc_tool_test.cc | 55 ++++++++++++++ 2 files changed, 175 insertions(+), 5 deletions(-) diff --git a/test/cpp/util/grpc_tool.cc b/test/cpp/util/grpc_tool.cc index bb6f8780202..7bf1a595e57 100644 --- a/test/cpp/util/grpc_tool.cc +++ b/test/cpp/util/grpc_tool.cc @@ -58,6 +58,11 @@ DEFINE_string(protofiles, "", "Name of the proto file."); DEFINE_bool(binary_input, false, "Input in binary format"); DEFINE_bool(binary_output, false, "Output in binary format"); DEFINE_string(infile, "", "Input file (default is stdin)"); +DEFINE_bool(batch, false, + "Input contains multiple requests. Please do not use this to send " + "more than a few RPCs. gRPC CLI has very different performance " + "characteristics compared with normal RPC calls which make it " + "unsuitable for loadtesting or significant production traffic."); namespace { @@ -460,12 +465,17 @@ bool GrpcTool::CallMethod(int argc, const char** argv, return false; } + if (argc == 3) { + request_text = argv[2]; + } + if (parser->IsStreaming(method_name, true /* is_request */)) { std::istream* input_stream; std::ifstream input_file; - if (argc == 3) { - request_text = argv[2]; + if (FLAGS_batch) { + fprintf(stderr, "Batch mode for streaming RPC is not supported.\n"); + return false; } std::multimap<grpc::string, grpc::string> client_metadata; @@ -549,8 +559,115 @@ bool GrpcTool::CallMethod(int argc, const char** argv, } } else { // parser->IsStreaming(method_name, true /* is_request */) + if (FLAGS_batch) { + if (parser->IsStreaming(method_name, false /* is_request */)) { + fprintf(stderr, "Batch mode for streaming RPC is not supported.\n"); + return false; + } + + std::istream* input_stream; + std::ifstream input_file; + + if (FLAGS_infile.empty()) { + if (isatty(STDIN_FILENO)) { + print_mode = true; + fprintf(stderr, "reading request messages from stdin...\n"); + } + input_stream = &std::cin; + } else { + input_file.open(FLAGS_infile, std::ios::in | std::ios::binary); + input_stream = &input_file; + } + + std::multimap<grpc::string, grpc::string> client_metadata; + ParseMetadataFlag(&client_metadata); + if (print_mode) { + PrintMetadata(client_metadata, "Sending client initial metadata:"); + } + + std::stringstream request_ss; + grpc::string line; + while (!request_text.empty() || + (!input_stream->eof() && getline(*input_stream, line))) { + if (!request_text.empty()) { + if (FLAGS_binary_input) { + serialized_request_proto = request_text; + request_text.clear(); + } else { + serialized_request_proto = parser->GetSerializedProtoFromMethod( + method_name, request_text, true /* is_request */); + request_text.clear(); + if (parser->HasError()) { + if (print_mode) { + fprintf(stderr, "Failed to parse request.\n"); + } + continue; + } + } + + grpc::string serialized_response_proto; + std::multimap<grpc::string_ref, grpc::string_ref> + server_initial_metadata, server_trailing_metadata; + CliCall call(channel, formatted_method_name, client_metadata); + call.Write(serialized_request_proto); + call.WritesDone(); + if (!call.Read(&serialized_response_proto, + &server_initial_metadata)) { + fprintf(stderr, "Failed to read response.\n"); + } + Status status = call.Finish(&server_trailing_metadata); + + if (status.ok()) { + if (print_mode) { + fprintf(stderr, "Rpc succeeded with OK status.\n"); + PrintMetadata(server_initial_metadata, + "Received initial metadata from server:"); + PrintMetadata(server_trailing_metadata, + "Received trailing metadata from server:"); + } + + if (FLAGS_binary_output) { + if (!callback(serialized_response_proto)) { + break; + } + } else { + grpc::string response_text = parser->GetTextFormatFromMethod( + method_name, serialized_response_proto, + false /* is_request */); + if (parser->HasError() && print_mode) { + fprintf(stderr, "Failed to parse response.\n"); + } else { + if (!callback(response_text)) { + break; + } + } + } + } else { + if (print_mode) { + fprintf(stderr, + "Rpc failed with status code %d, error message: %s\n", + status.error_code(), status.error_message().c_str()); + } + } + } else { + if (line.length() == 0) { + request_text = request_ss.str(); + request_ss.str(grpc::string()); + request_ss.clear(); + } else { + request_ss << line << ' '; + } + } + } + + if (input_file.is_open()) { + input_file.close(); + } + + return true; + } + if (argc == 3) { - request_text = argv[2]; if (!FLAGS_infile.empty()) { fprintf(stderr, "warning: request given in argv, ignoring --infile\n"); } @@ -571,9 +688,7 @@ bool GrpcTool::CallMethod(int argc, const char** argv, if (FLAGS_binary_input) { serialized_request_proto = request_text; - // formatted_method_name = method_name; } else { - // formatted_method_name = parser->GetFormattedMethodName(method_name); serialized_request_proto = parser->GetSerializedProtoFromMethod( method_name, request_text, true /* is_request */); if (parser->HasError()) { diff --git a/test/cpp/util/grpc_tool_test.cc b/test/cpp/util/grpc_tool_test.cc index dd00581f2b8..d0b3d7b81ba 100644 --- a/test/cpp/util/grpc_tool_test.cc +++ b/test/cpp/util/grpc_tool_test.cc @@ -84,6 +84,7 @@ namespace testing { DECLARE_bool(binary_input); DECLARE_bool(binary_output); DECLARE_bool(l); +DECLARE_bool(batch); namespace { @@ -399,6 +400,60 @@ TEST_F(GrpcToolTest, CallCommand) { ShutdownServer(); } +TEST_F(GrpcToolTest, CallCommandBatch) { + // Test input "grpc_cli call Echo" + std::stringstream output_stream; + + const grpc::string server_address = SetUpServer(); + const char* argv[] = {"grpc_cli", "call", server_address.c_str(), "Echo", + "message: 'Hello0'"}; + + // Mock std::cin input "message: 'Hello1'\n\n message: 'Hello2'\n\n" + std::streambuf* orig = std::cin.rdbuf(); + std::istringstream ss("message: 'Hello1'\n\n message: 'Hello2'\n\n"); + std::cin.rdbuf(ss.rdbuf()); + + FLAGS_batch = true; + EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, TestCliCredentials(), + std::bind(PrintStream, &output_stream, + std::placeholders::_1))); + FLAGS_batch = false; + + // Expected output: "message: "Hello0"\nmessage: "Hello1"\nmessage: + // "Hello2"\n" + EXPECT_TRUE(NULL != strstr(output_stream.str().c_str(), + "message: \"Hello0\"\nmessage: " + "\"Hello1\"\nmessage: \"Hello2\"\n")); + std::cin.rdbuf(orig); + ShutdownServer(); +} + +TEST_F(GrpcToolTest, CallCommandBatchWithBadRequest) { + // Test input "grpc_cli call Echo" + std::stringstream output_stream; + + const grpc::string server_address = SetUpServer(); + const char* argv[] = {"grpc_cli", "call", server_address.c_str(), "Echo", + "message: 'Hello0'"}; + + // Mock std::cin input "message: 1\n\n message: 'Hello2'\n\n" + std::streambuf* orig = std::cin.rdbuf(); + std::istringstream ss("message: 1\n\n message: 'Hello2'\n\n"); + std::cin.rdbuf(ss.rdbuf()); + + FLAGS_batch = true; + EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, TestCliCredentials(), + std::bind(PrintStream, &output_stream, + std::placeholders::_1))); + FLAGS_batch = false; + + // Expected output: "message: "Hello0"\nmessage: "Hello2"\n" + EXPECT_TRUE(NULL != strstr(output_stream.str().c_str(), + "message: \"Hello0\"\nmessage: \"Hello2\"\n")); + std::cin.rdbuf(orig); + ShutdownServer(); +} + TEST_F(GrpcToolTest, CallCommandRequestStream) { // Test input: grpc_cli call localhost:<port> RequestStream "message: // 'Hello0'" From 5dd2f48e242cd7ce788b84a6b07612098b4d96e3 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng <zyc@google.com> Date: Fri, 29 Sep 2017 14:52:33 -0700 Subject: [PATCH 11/20] Fix windows build --- test/cpp/util/grpc_tool.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cpp/util/grpc_tool.cc b/test/cpp/util/grpc_tool.cc index 7bf1a595e57..cd6084ac6dc 100644 --- a/test/cpp/util/grpc_tool.cc +++ b/test/cpp/util/grpc_tool.cc @@ -569,7 +569,7 @@ bool GrpcTool::CallMethod(int argc, const char** argv, std::ifstream input_file; if (FLAGS_infile.empty()) { - if (isatty(STDIN_FILENO)) { + if (isatty(fileno(stdin))) { print_mode = true; fprintf(stderr, "reading request messages from stdin...\n"); } From 3771adebb9fc03b3dfb1486614b6120c376aa123 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Vo=C3=9F?= <thomas.voss.bochum@gmail.com> Date: Sat, 30 Sep 2017 13:01:42 +0200 Subject: [PATCH 12/20] Fix up whitespace --- src/core/lib/iomgr/tcp_client_uv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/tcp_client_uv.c b/src/core/lib/iomgr/tcp_client_uv.c index f2b23aae2e6..0d9e7ed5f6e 100644 --- a/src/core/lib/iomgr/tcp_client_uv.c +++ b/src/core/lib/iomgr/tcp_client_uv.c @@ -145,7 +145,7 @@ static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx, connect->resource_quota = resource_quota; uv_tcp_init(uv_default_loop(), connect->tcp_handle); connect->connect_req.data = connect; - connect->refs = 2; // One for the connect operation, one for the timer. + connect->refs = 2; // One for the connect operation, one for the timer. if (GRPC_TRACER_ON(grpc_tcp_trace)) { gpr_log(GPR_DEBUG, "CLIENT_CONNECT: %s: asynchronously connecting", From 4c64dc9af87ba1826c4cbeef887b69663c5966c0 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch <jtattermusch@google.com> Date: Wed, 27 Sep 2017 16:53:47 +0200 Subject: [PATCH 13/20] workaround verconf.h problem --- tools/distrib/build_ruby_environment_macos.sh | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tools/distrib/build_ruby_environment_macos.sh b/tools/distrib/build_ruby_environment_macos.sh index c2240ce9761..fe0c5a4d70b 100644 --- a/tools/distrib/build_ruby_environment_macos.sh +++ b/tools/distrib/build_ruby_environment_macos.sh @@ -21,9 +21,10 @@ CROSS_RUBY=`mktemp tmpfile.XXXXXXXX` curl https://raw.githubusercontent.com/rake-compiler/rake-compiler/v1.0.3/tasks/bin/cross-ruby.rake > $CROSS_RUBY +# See https://github.com/grpc/grpc/issues/12161 for verconf.h patch details patch $CROSS_RUBY << EOF ---- cross-ruby.rake 2016-02-05 16:26:53.000000000 -0800 -+++ cross-ruby.rake.patched 2016-02-05 16:27:33.000000000 -0800 +--- cross-ruby.rake 2017-09-27 16:46:00.311020325 +0200 ++++ patched 2017-09-27 16:49:46.127016895 +0200 @@ -133,7 +133,8 @@ "--host=#{MINGW_HOST}", "--target=#{MINGW_TARGET}", @@ -32,8 +33,16 @@ patch $CROSS_RUBY << EOF + '--enable-static', + '--disable-shared', '--disable-install-doc', - '--without-tk', - '--without-tcl' + '--with-ext=' + ] +@@ -151,6 +152,7 @@ + # make + file "#{USER_HOME}/builds/#{MINGW_HOST}/#{RUBY_CC_VERSION}/ruby.exe" => ["#{USER_HOME}/builds/#{MINGW_HOST}/#{RUBY_CC_VERSION}/Makefile"] do |t| + chdir File.dirname(t.prerequisites.first) do ++ sh "test -s verconf.h || rm -f verconf.h" # if verconf.h has size 0, make sure it gets re-built by make + sh MAKE + end + end EOF MAKE="make -j8" From 6bb6fd9d1b7f45efe3faed36876cbb52a4b22fe1 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch <jtattermusch@google.com> Date: Wed, 27 Sep 2017 16:56:26 +0200 Subject: [PATCH 14/20] simplify macos artifact build script --- tools/internal_ci/macos/grpc_build_artifacts.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/internal_ci/macos/grpc_build_artifacts.sh b/tools/internal_ci/macos/grpc_build_artifacts.sh index 09784e3bb47..eb4568c32b1 100755 --- a/tools/internal_ci/macos/grpc_build_artifacts.sh +++ b/tools/internal_ci/macos/grpc_build_artifacts.sh @@ -27,8 +27,7 @@ python3.5 -m pip install cython setuptools wheel python3.6 -m pip install cython setuptools wheel # needed to build ruby artifacts -wget https://raw.githubusercontent.com/grpc/grpc/master/tools/distrib/build_ruby_environment_macos.sh -bash build_ruby_environment_macos.sh +time bash tools/distrib/build_ruby_environment_macos.sh gem install rubygems-update update_rubygems From 64e48b4fdb9a40b9507865ae4bdde4a4fffbffe4 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch <jtattermusch@google.com> Date: Mon, 2 Oct 2017 11:35:01 +0200 Subject: [PATCH 15/20] coreutils already installed --- tools/internal_ci/helper_scripts/prepare_build_macos_rc | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc index bec529f85e9..c785779b515 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc @@ -51,7 +51,6 @@ export LANG=en_US.UTF-8 pod repo update # needed by python # python -brew install coreutils # we need grealpath pip install virtualenv --user python pip install -U six tox setuptools twisted pyyaml --user python export PYTHONPATH=/Library/Python/3.4/site-packages From 15ccb3d38afa58981930a546aa1eab489b7bfacb Mon Sep 17 00:00:00 2001 From: Jan Tattermusch <jtattermusch@google.com> Date: Mon, 2 Oct 2017 17:53:49 +0200 Subject: [PATCH 16/20] debug clock skew --- tools/internal_ci/helper_scripts/prepare_build_macos_rc | 3 +++ tools/internal_ci/macos/grpc_run_tests_matrix.sh | 3 +++ 2 files changed, 6 insertions(+) diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc index bec529f85e9..9cb24675101 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc @@ -59,4 +59,7 @@ export PYTHONPATH=/Library/Python/3.4/site-packages # set xcode version for Obj-C tests sudo xcode-select -switch /Applications/Xcode_8.2.1.app/Contents/Developer +# TODO(jtattermusch): better debugging of clock skew, remove once not needed +date + git submodule update --init diff --git a/tools/internal_ci/macos/grpc_run_tests_matrix.sh b/tools/internal_ci/macos/grpc_run_tests_matrix.sh index 8e7fd54a629..6e0c2bb4874 100755 --- a/tools/internal_ci/macos/grpc_run_tests_matrix.sh +++ b/tools/internal_ci/macos/grpc_run_tests_matrix.sh @@ -28,6 +28,9 @@ ps aux | grep port_server\\.py | awk '{print $2}' | xargs kill -9 # Reveal leftover processes that might be left behind by the build ps aux | grep -i kbuilder +# TODO(jtattermusch): better debugging of clock skew, remove once not needed +date + echo 'Exiting gRPC main test script.' if [ "$FAILED" != "" ] From 89cf5a409355ec4dd2083da0fcf01e964606d219 Mon Sep 17 00:00:00 2001 From: Muxi Yan <mxyan@google.com> Date: Mon, 2 Oct 2017 13:19:56 -0700 Subject: [PATCH 17/20] Eliminate gRPC-Core umbrella header warning by tweaking module.modulemap and excluding some files for other build systems --- gRPC-Core.podspec | 11 ---- include/grpc/module.modulemap | 63 ++++++++++++++++++- templates/gRPC-Core.podspec.template | 10 ++- .../include/grpc/module.modulemap.template | 32 ++++++++++ 4 files changed, 101 insertions(+), 15 deletions(-) create mode 100644 templates/include/grpc/module.modulemap.template diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index c76b9a3f4bd..d34c271dbaf 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -105,8 +105,6 @@ Pod::Spec.new do |s| ss.source_files = 'include/grpc/support/alloc.h', 'include/grpc/support/atm.h', 'include/grpc/support/atm_gcc_atomic.h', - 'include/grpc/support/atm_gcc_sync.h', - 'include/grpc/support/atm_windows.h', 'include/grpc/support/avl.h', 'include/grpc/support/cmdline.h', 'include/grpc/support/cpu.h', @@ -121,18 +119,13 @@ Pod::Spec.new do |s| 'include/grpc/support/sync_custom.h', 'include/grpc/support/sync_generic.h', 'include/grpc/support/sync_posix.h', - 'include/grpc/support/sync_windows.h', 'include/grpc/support/thd.h', 'include/grpc/support/time.h', 'include/grpc/support/tls.h', - 'include/grpc/support/tls_gcc.h', - 'include/grpc/support/tls_msvc.h', 'include/grpc/support/tls_pthread.h', 'include/grpc/support/useful.h', 'include/grpc/impl/codegen/atm.h', 'include/grpc/impl/codegen/atm_gcc_atomic.h', - 'include/grpc/impl/codegen/atm_gcc_sync.h', - 'include/grpc/impl/codegen/atm_windows.h', 'include/grpc/impl/codegen/gpr_slice.h', 'include/grpc/impl/codegen/gpr_types.h', 'include/grpc/impl/codegen/port_platform.h', @@ -140,7 +133,6 @@ Pod::Spec.new do |s| 'include/grpc/impl/codegen/sync_custom.h', 'include/grpc/impl/codegen/sync_generic.h', 'include/grpc/impl/codegen/sync_posix.h', - 'include/grpc/impl/codegen/sync_windows.h', 'include/grpc/impl/codegen/byte_buffer.h', 'include/grpc/impl/codegen/byte_buffer_reader.h', 'include/grpc/impl/codegen/compression_types.h', @@ -152,8 +144,6 @@ Pod::Spec.new do |s| 'include/grpc/impl/codegen/status.h', 'include/grpc/impl/codegen/atm.h', 'include/grpc/impl/codegen/atm_gcc_atomic.h', - 'include/grpc/impl/codegen/atm_gcc_sync.h', - 'include/grpc/impl/codegen/atm_windows.h', 'include/grpc/impl/codegen/gpr_slice.h', 'include/grpc/impl/codegen/gpr_types.h', 'include/grpc/impl/codegen/port_platform.h', @@ -161,7 +151,6 @@ Pod::Spec.new do |s| 'include/grpc/impl/codegen/sync_custom.h', 'include/grpc/impl/codegen/sync_generic.h', 'include/grpc/impl/codegen/sync_posix.h', - 'include/grpc/impl/codegen/sync_windows.h', 'include/grpc/grpc_security.h', 'include/grpc/byte_buffer.h', 'include/grpc/byte_buffer_reader.h', diff --git a/include/grpc/module.modulemap b/include/grpc/module.modulemap index 51bfef2cc93..226cc6cf871 100644 --- a/include/grpc/module.modulemap +++ b/include/grpc/module.modulemap @@ -1,12 +1,69 @@ + framework module grpc { umbrella header "grpc.h" - header "byte_buffer_reader.h" - header "grpc_security.h" - header "grpc_security_constants.h" header "support/alloc.h" + header "support/atm.h" + header "support/atm_gcc_atomic.h" + header "support/avl.h" + header "support/cmdline.h" + header "support/cpu.h" + header "support/histogram.h" + header "support/host_port.h" + header "support/log.h" + header "support/log_windows.h" header "support/port_platform.h" header "support/string_util.h" + header "support/subprocess.h" + header "support/sync.h" + header "support/sync_custom.h" + header "support/sync_generic.h" + header "support/sync_posix.h" + header "support/thd.h" + header "support/time.h" + header "support/tls.h" + header "support/tls_pthread.h" + header "support/useful.h" + header "impl/codegen/atm.h" + header "impl/codegen/atm_gcc_atomic.h" + header "impl/codegen/gpr_slice.h" + header "impl/codegen/gpr_types.h" + header "impl/codegen/port_platform.h" + header "impl/codegen/sync.h" + header "impl/codegen/sync_custom.h" + header "impl/codegen/sync_generic.h" + header "impl/codegen/sync_posix.h" + header "impl/codegen/byte_buffer.h" + header "impl/codegen/byte_buffer_reader.h" + header "impl/codegen/compression_types.h" + header "impl/codegen/connectivity_state.h" + header "impl/codegen/exec_ctx_fwd.h" + header "impl/codegen/grpc_types.h" + header "impl/codegen/propagation_bits.h" + header "impl/codegen/slice.h" + header "impl/codegen/status.h" + header "impl/codegen/atm.h" + header "impl/codegen/atm_gcc_atomic.h" + header "impl/codegen/gpr_slice.h" + header "impl/codegen/gpr_types.h" + header "impl/codegen/port_platform.h" + header "impl/codegen/sync.h" + header "impl/codegen/sync_custom.h" + header "impl/codegen/sync_generic.h" + header "impl/codegen/sync_posix.h" + header "grpc_security.h" + header "byte_buffer.h" + header "byte_buffer_reader.h" + header "compression.h" + header "grpc.h" + header "grpc_posix.h" + header "grpc_security_constants.h" + header "load_reporting.h" + header "slice.h" + header "slice_buffer.h" + header "status.h" + header "support/workaround_list.h" + header "census.h" export * module * { export * } diff --git a/templates/gRPC-Core.podspec.template b/templates/gRPC-Core.podspec.template index 6077f8098dd..22814849172 100644 --- a/templates/gRPC-Core.podspec.template +++ b/templates/gRPC-Core.podspec.template @@ -31,11 +31,19 @@ return [f for f in out if not f.startswith("third_party/nanopb/")] def grpc_public_headers(libs): + excluded_files = ["include/grpc/support/atm_gcc_sync.h", + "include/grpc/support/atm_windows.h", + "include/grpc/support/sync_windows.h", + "include/grpc/support/tls_gcc.h", + "include/grpc/support/tls_msvc.h", + "include/grpc/impl/codegen/atm_gcc_sync.h", + "include/grpc/impl/codegen/atm_windows.h", + "include/grpc/impl/codegen/sync_windows.h"] out = [] for lib in libs: if lib.name in ("grpc", "gpr"): out += lib.get('public_headers', []) - return out + return [f for f in out if not f in excluded_files] def grpc_private_headers(libs): out = [] diff --git a/templates/include/grpc/module.modulemap.template b/templates/include/grpc/module.modulemap.template new file mode 100644 index 00000000000..8edc4b56acd --- /dev/null +++ b/templates/include/grpc/module.modulemap.template @@ -0,0 +1,32 @@ +%YAML 1.2 +--- | + <%! + def grpc_public_headers_no_dir(libs): + excluded_files = ["include/grpc/support/atm_gcc_sync.h", + "include/grpc/support/atm_windows.h", + "include/grpc/support/sync_windows.h", + "include/grpc/support/tls_gcc.h", + "include/grpc/support/tls_msvc.h", + "include/grpc/impl/codegen/atm_gcc_sync.h", + "include/grpc/impl/codegen/atm_windows.h", + "include/grpc/impl/codegen/sync_windows.h"] + out = [] + for lib in libs: + if lib.name in ("grpc", "gpr"): + out += lib.get('public_headers', []) + out = [f for f in out if f not in excluded_files] + out = [hdr.split('/', 2)[2] for hdr in out] + return out + + def header_lines(files): + return ('\n ').join('header "%s"' % f for f in files) + %> + framework module grpc { + umbrella header "grpc.h" + + ${header_lines(grpc_public_headers_no_dir(libs))} + + export * + module * { export * } + } + From 9887379158b5418d983e79bc1bc8b4ba8c72bd54 Mon Sep 17 00:00:00 2001 From: murgatroid99 <mlumish@google.com> Date: Mon, 2 Oct 2017 17:28:27 -0700 Subject: [PATCH 18/20] Update version to 1.8.0-dev, update g word to 'generous' --- BUILD | 4 ++-- CMakeLists.txt | 2 +- Makefile | 4 ++-- build.yaml | 4 ++-- doc/g_stands_for.md | 1 + gRPC-Core.podspec | 2 +- gRPC-ProtoRPC.podspec | 2 +- gRPC-RxLibrary.podspec | 2 +- gRPC.podspec | 2 +- package.json | 2 +- package.xml | 4 ++-- src/core/lib/surface/version.c | 2 +- src/cpp/common/version_cc.cc | 2 +- src/csharp/Grpc.Core/Version.csproj.include | 2 +- src/csharp/Grpc.Core/VersionInfo.cs | 4 ++-- src/csharp/build_packages_dotnetcli.bat | 2 +- src/csharp/build_packages_dotnetcli.sh | 4 ++-- src/node/health_check/package.json | 4 ++-- src/node/tools/package.json | 2 +- src/objective-c/!ProtoCompiler-gRPCPlugin.podspec | 2 +- src/objective-c/GRPCClient/private/version.h | 2 +- src/php/composer.json | 2 +- src/php/ext/grpc/version.h | 2 +- src/python/grpcio/grpc/_grpcio_metadata.py | 2 +- src/python/grpcio/grpc_version.py | 2 +- src/python/grpcio_health_checking/grpc_version.py | 2 +- src/python/grpcio_reflection/grpc_version.py | 2 +- src/python/grpcio_testing/grpc_version.py | 2 +- src/python/grpcio_tests/grpc_version.py | 2 +- src/ruby/lib/grpc/version.rb | 2 +- src/ruby/tools/version.rb | 2 +- tools/distrib/python/grpcio_tools/grpc_version.py | 2 +- tools/doxygen/Doxyfile.c++ | 2 +- tools/doxygen/Doxyfile.c++.internal | 2 +- 34 files changed, 41 insertions(+), 40 deletions(-) diff --git a/BUILD b/BUILD index dfcded21976..79e3aeb8491 100644 --- a/BUILD +++ b/BUILD @@ -34,11 +34,11 @@ load( ) # This should be updated along with build.yaml -g_stands_for = "gambit" +g_stands_for = "generous" core_version = "4.0.0-dev" -version = "1.7.0-dev" +version = "1.8.0-dev" GPR_PUBLIC_HDRS = [ "include/grpc/support/alloc.h", diff --git a/CMakeLists.txt b/CMakeLists.txt index 2e448c92b66..e92e19465bc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,7 +24,7 @@ cmake_minimum_required(VERSION 2.8) set(PACKAGE_NAME "grpc") -set(PACKAGE_VERSION "1.7.0-dev") +set(PACKAGE_VERSION "1.8.0-dev") set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}") set(PACKAGE_TARNAME "${PACKAGE_NAME}-${PACKAGE_VERSION}") set(PACKAGE_BUGREPORT "https://github.com/grpc/grpc/issues/") diff --git a/Makefile b/Makefile index 517ddfd90e0..22520ac9a15 100644 --- a/Makefile +++ b/Makefile @@ -411,8 +411,8 @@ Q = @ endif CORE_VERSION = 5.0.0-dev -CPP_VERSION = 1.7.0-dev -CSHARP_VERSION = 1.7.0-dev +CPP_VERSION = 1.8.0-dev +CSHARP_VERSION = 1.8.0-dev CPPFLAGS_NO_ARCH += $(addprefix -I, $(INCLUDES)) $(addprefix -D, $(DEFINES)) CPPFLAGS += $(CPPFLAGS_NO_ARCH) $(ARCH_FLAGS) diff --git a/build.yaml b/build.yaml index 9b5aacc4c8f..7563d4bfc06 100644 --- a/build.yaml +++ b/build.yaml @@ -13,8 +13,8 @@ settings: '#09': Per-language overrides are possible with (eg) ruby_version tag here '#10': See the expand_version.py for all the quirks here core_version: 5.0.0-dev - g_stands_for: gambit - version: 1.7.0-dev + g_stands_for: generous + version: 1.8.0-dev filegroups: - name: census public_headers: diff --git a/doc/g_stands_for.md b/doc/g_stands_for.md index 9a161d67a30..4e2ca33276c 100644 --- a/doc/g_stands_for.md +++ b/doc/g_stands_for.md @@ -11,3 +11,4 @@ future), and the corresponding version numbers that used them: - 1.4 'g' stands for 'gregarious' - 1.6 'g' stands for 'garcia' - 1.7 'g' stands for 'gambit' +- 1.8 'g' stands for 'generous' diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index d34c271dbaf..7b0c72ea1a1 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -22,7 +22,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-Core' - version = '1.7.0-dev' + version = '1.8.0-dev' s.version = version s.summary = 'Core cross-platform gRPC library, written in C' s.homepage = 'https://grpc.io' diff --git a/gRPC-ProtoRPC.podspec b/gRPC-ProtoRPC.podspec index cb40330dab5..db1e8db06c0 100644 --- a/gRPC-ProtoRPC.podspec +++ b/gRPC-ProtoRPC.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-ProtoRPC' - version = '1.7.0-dev' + version = '1.8.0-dev' s.version = version s.summary = 'RPC library for Protocol Buffers, based on gRPC' s.homepage = 'https://grpc.io' diff --git a/gRPC-RxLibrary.podspec b/gRPC-RxLibrary.podspec index 52bd8ed0ae1..36897790395 100644 --- a/gRPC-RxLibrary.podspec +++ b/gRPC-RxLibrary.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-RxLibrary' - version = '1.7.0-dev' + version = '1.8.0-dev' s.version = version s.summary = 'Reactive Extensions library for iOS/OSX.' s.homepage = 'https://grpc.io' diff --git a/gRPC.podspec b/gRPC.podspec index 79315e46f31..6bec50f0de3 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -20,7 +20,7 @@ Pod::Spec.new do |s| s.name = 'gRPC' - version = '1.7.0-dev' + version = '1.8.0-dev' s.version = version s.summary = 'gRPC client library for iOS/OSX' s.homepage = 'https://grpc.io' diff --git a/package.json b/package.json index 2e31275bf02..a0f0ff1710d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "grpc", - "version": "1.7.0-dev", + "version": "1.8.0-dev", "author": "Google Inc.", "description": "gRPC Library for Node", "homepage": "https://grpc.io/", diff --git a/package.xml b/package.xml index 0eea122b24e..dacb2d3d9a6 100644 --- a/package.xml +++ b/package.xml @@ -13,8 +13,8 @@ <date>2017-08-24</date> <time>16:06:07</time> <version> - <release>1.7.0dev</release> - <api>1.7.0dev</api> + <release>1.8.0dev</release> + <api>1.8.0dev</api> </version> <stability> <release>beta</release> diff --git a/src/core/lib/surface/version.c b/src/core/lib/surface/version.c index fd6ea4daa9f..6cb8e7e1a0d 100644 --- a/src/core/lib/surface/version.c +++ b/src/core/lib/surface/version.c @@ -23,4 +23,4 @@ const char *grpc_version_string(void) { return "5.0.0-dev"; } -const char *grpc_g_stands_for(void) { return "gambit"; } +const char *grpc_g_stands_for(void) { return "generous"; } diff --git a/src/cpp/common/version_cc.cc b/src/cpp/common/version_cc.cc index 2e9a51316d4..8049cbe0c90 100644 --- a/src/cpp/common/version_cc.cc +++ b/src/cpp/common/version_cc.cc @@ -22,5 +22,5 @@ #include <grpc++/grpc++.h> namespace grpc { -grpc::string Version() { return "1.7.0-dev"; } +grpc::string Version() { return "1.8.0-dev"; } } diff --git a/src/csharp/Grpc.Core/Version.csproj.include b/src/csharp/Grpc.Core/Version.csproj.include index 124ecab14cd..b9ceaf82543 100755 --- a/src/csharp/Grpc.Core/Version.csproj.include +++ b/src/csharp/Grpc.Core/Version.csproj.include @@ -1,7 +1,7 @@ <!-- This file is generated --> <Project> <PropertyGroup> - <GrpcCsharpVersion>1.7.0-dev</GrpcCsharpVersion> + <GrpcCsharpVersion>1.8.0-dev</GrpcCsharpVersion> <GoogleProtobufVersion>3.3.0</GoogleProtobufVersion> </PropertyGroup> </Project> diff --git a/src/csharp/Grpc.Core/VersionInfo.cs b/src/csharp/Grpc.Core/VersionInfo.cs index 588cc845165..dab938821fa 100644 --- a/src/csharp/Grpc.Core/VersionInfo.cs +++ b/src/csharp/Grpc.Core/VersionInfo.cs @@ -33,11 +33,11 @@ namespace Grpc.Core /// <summary> /// Current <c>AssemblyFileVersion</c> of gRPC C# assemblies /// </summary> - public const string CurrentAssemblyFileVersion = "1.7.0.0"; + public const string CurrentAssemblyFileVersion = "1.8.0.0"; /// <summary> /// Current version of gRPC C# /// </summary> - public const string CurrentVersion = "1.7.0-dev"; + public const string CurrentVersion = "1.8.0-dev"; } } diff --git a/src/csharp/build_packages_dotnetcli.bat b/src/csharp/build_packages_dotnetcli.bat index c419d870492..ff013d56800 100755 --- a/src/csharp/build_packages_dotnetcli.bat +++ b/src/csharp/build_packages_dotnetcli.bat @@ -13,7 +13,7 @@ @rem limitations under the License. @rem Current package versions -set VERSION=1.7.0-dev +set VERSION=1.8.0-dev @rem Adjust the location of nuget.exe set NUGET=C:\nuget\nuget.exe diff --git a/src/csharp/build_packages_dotnetcli.sh b/src/csharp/build_packages_dotnetcli.sh index 124dfbb257e..44a4791146b 100755 --- a/src/csharp/build_packages_dotnetcli.sh +++ b/src/csharp/build_packages_dotnetcli.sh @@ -39,7 +39,7 @@ dotnet pack --configuration Release Grpc.Auth --output ../../../artifacts dotnet pack --configuration Release Grpc.HealthCheck --output ../../../artifacts dotnet pack --configuration Release Grpc.Reflection --output ../../../artifacts -nuget pack Grpc.nuspec -Version "1.7.0-dev" -OutputDirectory ../../artifacts -nuget pack Grpc.Tools.nuspec -Version "1.7.0-dev" -OutputDirectory ../../artifacts +nuget pack Grpc.nuspec -Version "1.8.0-dev" -OutputDirectory ../../artifacts +nuget pack Grpc.Tools.nuspec -Version "1.8.0-dev" -OutputDirectory ../../artifacts (cd ../../artifacts && zip csharp_nugets_dotnetcli.zip *.nupkg) diff --git a/src/node/health_check/package.json b/src/node/health_check/package.json index 3c7d3707ee0..6f09c8f9f09 100644 --- a/src/node/health_check/package.json +++ b/src/node/health_check/package.json @@ -1,6 +1,6 @@ { "name": "grpc-health-check", - "version": "1.7.0-dev", + "version": "1.8.0-dev", "author": "Google Inc.", "description": "Health check service for use with gRPC", "repository": { @@ -15,7 +15,7 @@ } ], "dependencies": { - "grpc": "^1.7.0-dev", + "grpc": "^1.8.0-dev", "lodash": "^3.9.3", "google-protobuf": "^3.0.0" }, diff --git a/src/node/tools/package.json b/src/node/tools/package.json index d9b1fb86c91..f88fc65cdf4 100644 --- a/src/node/tools/package.json +++ b/src/node/tools/package.json @@ -1,6 +1,6 @@ { "name": "grpc-tools", - "version": "1.7.0-dev", + "version": "1.8.0-dev", "author": "Google Inc.", "description": "Tools for developing with gRPC on Node.js", "homepage": "https://grpc.io/", diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index 7d073c9a848..9065ab9f73e 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -42,7 +42,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler-gRPCPlugin' - v = '1.7.0-dev' + v = '1.8.0-dev' s.version = v s.summary = 'The gRPC ProtoC plugin generates Objective-C files from .proto services.' s.description = <<-DESC diff --git a/src/objective-c/GRPCClient/private/version.h b/src/objective-c/GRPCClient/private/version.h index 843954e84a0..db589d12de6 100644 --- a/src/objective-c/GRPCClient/private/version.h +++ b/src/objective-c/GRPCClient/private/version.h @@ -23,4 +23,4 @@ // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.7.0-dev" +#define GRPC_OBJC_VERSION_STRING @"1.8.0-dev" diff --git a/src/php/composer.json b/src/php/composer.json index 3606a18f341..09471d23fee 100644 --- a/src/php/composer.json +++ b/src/php/composer.json @@ -2,7 +2,7 @@ "name": "grpc/grpc-dev", "description": "gRPC library for PHP - for Developement use only", "license": "Apache-2.0", - "version": "1.7.0", + "version": "1.8.0", "require": { "php": ">=5.5.0", "google/protobuf": "^v3.3.0" diff --git a/src/php/ext/grpc/version.h b/src/php/ext/grpc/version.h index 07d8eee7fe2..93dd563cffb 100644 --- a/src/php/ext/grpc/version.h +++ b/src/php/ext/grpc/version.h @@ -20,6 +20,6 @@ #ifndef VERSION_H #define VERSION_H -#define PHP_GRPC_VERSION "1.7.0dev" +#define PHP_GRPC_VERSION "1.8.0dev" #endif /* VERSION_H */ diff --git a/src/python/grpcio/grpc/_grpcio_metadata.py b/src/python/grpcio/grpc/_grpcio_metadata.py index a4eb358c4ee..0887ac17224 100644 --- a/src/python/grpcio/grpc/_grpcio_metadata.py +++ b/src/python/grpcio/grpc/_grpcio_metadata.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc/_grpcio_metadata.py.template`!!! -__version__ = """1.7.0.dev0""" +__version__ = """1.8.0.dev0""" diff --git a/src/python/grpcio/grpc_version.py b/src/python/grpcio/grpc_version.py index 3194a893d7e..61c41573756 100644 --- a/src/python/grpcio/grpc_version.py +++ b/src/python/grpcio/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_version.py.template`!!! -VERSION='1.7.0.dev0' +VERSION='1.8.0.dev0' diff --git a/src/python/grpcio_health_checking/grpc_version.py b/src/python/grpcio_health_checking/grpc_version.py index ef68bad17a0..889297f0209 100644 --- a/src/python/grpcio_health_checking/grpc_version.py +++ b/src/python/grpcio_health_checking/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_health_checking/grpc_version.py.template`!!! -VERSION='1.7.0.dev0' +VERSION='1.8.0.dev0' diff --git a/src/python/grpcio_reflection/grpc_version.py b/src/python/grpcio_reflection/grpc_version.py index 55ab959cc57..192f4cc2174 100644 --- a/src/python/grpcio_reflection/grpc_version.py +++ b/src/python/grpcio_reflection/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_reflection/grpc_version.py.template`!!! -VERSION='1.7.0.dev0' +VERSION='1.8.0.dev0' diff --git a/src/python/grpcio_testing/grpc_version.py b/src/python/grpcio_testing/grpc_version.py index 592d08efc36..83470c28253 100644 --- a/src/python/grpcio_testing/grpc_version.py +++ b/src/python/grpcio_testing/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_testing/grpc_version.py.template`!!! -VERSION='1.7.0.dev0' +VERSION='1.8.0.dev0' diff --git a/src/python/grpcio_tests/grpc_version.py b/src/python/grpcio_tests/grpc_version.py index 9e54dc9f75b..7065edd3bfc 100644 --- a/src/python/grpcio_tests/grpc_version.py +++ b/src/python/grpcio_tests/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_tests/grpc_version.py.template`!!! -VERSION='1.7.0.dev0' +VERSION='1.8.0.dev0' diff --git a/src/ruby/lib/grpc/version.rb b/src/ruby/lib/grpc/version.rb index 228c01a92c4..3001579ce77 100644 --- a/src/ruby/lib/grpc/version.rb +++ b/src/ruby/lib/grpc/version.rb @@ -14,5 +14,5 @@ # GRPC contains the General RPC module. module GRPC - VERSION = '1.7.0.dev' + VERSION = '1.8.0.dev' end diff --git a/src/ruby/tools/version.rb b/src/ruby/tools/version.rb index ea0c4ae56cc..c584a7cf593 100644 --- a/src/ruby/tools/version.rb +++ b/src/ruby/tools/version.rb @@ -14,6 +14,6 @@ module GRPC module Tools - VERSION = '1.7.0.dev' + VERSION = '1.8.0.dev' end end diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index a4178a58c1f..db92b10c603 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION='1.7.0.dev0' +VERSION='1.8.0.dev0' diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index d81b7b4d115..eb27eed075f 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.7.0-dev +PROJECT_NUMBER = 1.8.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index eacb40c2123..584dd0af57b 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.7.0-dev +PROJECT_NUMBER = 1.8.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a From 0bb3f2983fcffd4b044cbb3d67cf4b0d2ab177b3 Mon Sep 17 00:00:00 2001 From: murgatroid99 <mlumish@google.com> Date: Mon, 2 Oct 2017 17:39:58 -0700 Subject: [PATCH 19/20] Update core version in BUILD file --- BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BUILD b/BUILD index 79e3aeb8491..1063b74f068 100644 --- a/BUILD +++ b/BUILD @@ -36,7 +36,7 @@ load( # This should be updated along with build.yaml g_stands_for = "generous" -core_version = "4.0.0-dev" +core_version = "5.0.0-dev" version = "1.8.0-dev" From a36b429ae89c509400b893c617347ceb94471d4d Mon Sep 17 00:00:00 2001 From: Florian Nagel <hi@floriannagel.xyz> Date: Tue, 3 Oct 2017 15:21:14 +0200 Subject: [PATCH 20/20] Fix deprecation warning Server#addProtoService and use Server#addService instead --- examples/node/dynamic_codegen/greeter_server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/node/dynamic_codegen/greeter_server.js b/examples/node/dynamic_codegen/greeter_server.js index f9cb1b13d66..180f96c28b8 100644 --- a/examples/node/dynamic_codegen/greeter_server.js +++ b/examples/node/dynamic_codegen/greeter_server.js @@ -34,7 +34,7 @@ function sayHello(call, callback) { */ function main() { var server = new grpc.Server(); - server.addProtoService(hello_proto.Greeter.service, {sayHello: sayHello}); + server.addService(hello_proto.Greeter.service, {sayHello: sayHello}); server.bind('0.0.0.0:50051', grpc.ServerCredentials.createInsecure()); server.start(); }