From b65daaa52d2e24dadff465a2a84c0e8220282047 Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Thu, 13 Aug 2020 00:12:55 -0700 Subject: [PATCH 01/62] Copy channel args hash before appending ruby user agent --- src/ruby/lib/grpc/generic/client_stub.rb | 2 +- src/ruby/spec/user_agent_spec.rb | 74 ++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 src/ruby/spec/user_agent_spec.rb diff --git a/src/ruby/lib/grpc/generic/client_stub.rb b/src/ruby/lib/grpc/generic/client_stub.rb index b193f5c4e18..1884dcba246 100644 --- a/src/ruby/lib/grpc/generic/client_stub.rb +++ b/src/ruby/lib/grpc/generic/client_stub.rb @@ -100,7 +100,7 @@ module GRPC channel_args: {}, interceptors: []) @ch = ClientStub.setup_channel(channel_override, host, creds, - channel_args) + channel_args.dup) alt_host = channel_args[Core::Channel::SSL_TARGET] @host = alt_host.nil? ? host : alt_host @propagate_mask = propagate_mask diff --git a/src/ruby/spec/user_agent_spec.rb b/src/ruby/spec/user_agent_spec.rb new file mode 100644 index 00000000000..a5aa2066db1 --- /dev/null +++ b/src/ruby/spec/user_agent_spec.rb @@ -0,0 +1,74 @@ +# Copyright 2020 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'spec_helper' + +# a test service that checks the cert of its peer +class UserAgentEchoService + include GRPC::GenericService + rpc :an_rpc, EchoMsg, EchoMsg + + def an_rpc(_req, call) + EchoMsg.new(msg: call.metadata['user-agent']) + end +end + +UserAgentEchoServiceStub = UserAgentEchoService.rpc_stub_class + +describe 'user agent' do + RpcServer = GRPC::RpcServer + + before(:all) do + server_opts = { + poll_period: 1 + } + @srv = new_rpc_server_for_testing(**server_opts) + @port = @srv.add_http2_port('0.0.0.0:0', :this_port_is_insecure) + @srv.handle(UserAgentEchoService) + @srv_thd = Thread.new { @srv.run } + @srv.wait_till_running + end + + after(:all) do + expect(@srv.stopped?).to be(false) + @srv.stop + @srv_thd.join + end + + it 'client sends expected user agent' do + stub = UserAgentEchoServiceStub.new("localhost:#{@port}", + :this_channel_is_insecure, + {}) + response = stub.an_rpc(EchoMsg.new) + expected_user_agent_prefix = "grpc-ruby/#{GRPC::VERSION}" + expect(response.msg.start_with?(expected_user_agent_prefix)).to be true + # check that the expected user agent prefix occurs in the real user agent exactly once + expect(response.msg.split(expected_user_agent_prefix).size).to eq 2 + end + + it 'user agent header does not grow when the same channel args hash is used across multiple stubs' do + shared_channel_args_hash = {} + 10.times do + stub = UserAgentEchoServiceStub.new("localhost:#{@port}", + :this_channel_is_insecure, + channel_args: shared_channel_args_hash) + response = stub.an_rpc(EchoMsg.new) + puts "got echo response: #{response.msg}" + expected_user_agent_prefix = "grpc-ruby/#{GRPC::VERSION}" + expect(response.msg.start_with?(expected_user_agent_prefix)).to be true + # check that the expected user agent prefix occurs in the real user agent exactly once + expect(response.msg.split(expected_user_agent_prefix).size).to eq 2 + end + end +end From dd97cc0a88eb7aa668eef5f0766c9e2a5f66e6aa Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Thu, 13 Aug 2020 19:29:02 -0700 Subject: [PATCH 02/62] Bump 1.31.x to 1.31.1 --- BUILD | 2 +- build_handwritten.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/BUILD b/BUILD index ecb149a54ff..7b94d4a0fbc 100644 --- a/BUILD +++ b/BUILD @@ -79,7 +79,7 @@ g_stands_for = "galore" core_version = "11.0.0" -version = "1.31.0" +version = "1.31.1" GPR_PUBLIC_HDRS = [ "include/grpc/support/alloc.h", diff --git a/build_handwritten.yaml b/build_handwritten.yaml index 006ba4c5b80..45b3a4d29e5 100644 --- a/build_handwritten.yaml +++ b/build_handwritten.yaml @@ -15,7 +15,7 @@ settings: core_version: 11.0.0 csharp_major_version: 2 g_stands_for: galore - version: 1.31.0 + version: 1.31.1 targets: - name: check_epollexclusive build: tool From 011986e1fd1735dd8c2cb79fe02662f89ccd472b Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Thu, 13 Aug 2020 19:29:38 -0700 Subject: [PATCH 03/62] Regenerate projects --- CMakeLists.txt | 6 +++--- Makefile | 4 ++-- gRPC-C++.podspec | 2 +- gRPC-Core.podspec | 2 +- gRPC-ProtoRPC.podspec | 2 +- gRPC-RxLibrary.podspec | 2 +- gRPC.podspec | 2 +- package.xml | 6 +++--- src/cpp/common/version_cc.cc | 2 +- src/csharp/Grpc.Core.Api/VersionInfo.cs | 4 ++-- src/csharp/build/dependencies.props | 2 +- src/csharp/build_unitypackage.bat | 2 +- src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec | 2 +- src/objective-c/!ProtoCompiler-gRPCPlugin.podspec | 2 +- src/objective-c/GRPCClient/version.h | 2 +- src/objective-c/tests/version.h | 2 +- src/php/composer.json | 2 +- src/php/ext/grpc/version.h | 2 +- src/python/grpcio/grpc/_grpcio_metadata.py | 2 +- src/python/grpcio/grpc_version.py | 2 +- src/python/grpcio_channelz/grpc_version.py | 2 +- src/python/grpcio_health_checking/grpc_version.py | 2 +- src/python/grpcio_reflection/grpc_version.py | 2 +- src/python/grpcio_status/grpc_version.py | 2 +- src/python/grpcio_testing/grpc_version.py | 2 +- src/python/grpcio_tests/grpc_version.py | 2 +- src/ruby/lib/grpc/version.rb | 2 +- src/ruby/tools/version.rb | 2 +- tools/distrib/python/grpcio_tools/grpc_version.py | 2 +- tools/doxygen/Doxyfile.c++ | 2 +- tools/doxygen/Doxyfile.c++.internal | 2 +- tools/doxygen/Doxyfile.objc | 2 +- tools/doxygen/Doxyfile.objc.internal | 2 +- tools/doxygen/Doxyfile.php | 2 +- 34 files changed, 40 insertions(+), 40 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 33f76f073d7..a067308142f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,12 +25,12 @@ cmake_minimum_required(VERSION 3.5.1) set(PACKAGE_NAME "grpc") -set(PACKAGE_VERSION "1.31.0") +set(PACKAGE_VERSION "1.31.1") set(gRPC_CORE_VERSION "11.0.0") set(gRPC_CORE_SOVERSION "11") -set(gRPC_CPP_VERSION "1.31.0") +set(gRPC_CPP_VERSION "1.31.1") set(gRPC_CPP_SOVERSION "1") -set(gRPC_CSHARP_VERSION "2.31.0") +set(gRPC_CSHARP_VERSION "2.31.1") set(gRPC_CSHARP_SOVERSION "2") set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}") set(PACKAGE_TARNAME "${PACKAGE_NAME}-${PACKAGE_VERSION}") diff --git a/Makefile b/Makefile index ad36296ac8a..fdb1b849333 100644 --- a/Makefile +++ b/Makefile @@ -450,8 +450,8 @@ Q = @ endif CORE_VERSION = 11.0.0 -CPP_VERSION = 1.31.0 -CSHARP_VERSION = 2.31.0 +CPP_VERSION = 1.31.1 +CSHARP_VERSION = 2.31.1 CPPFLAGS_NO_ARCH += $(addprefix -I, $(INCLUDES)) $(addprefix -D, $(DEFINES)) CPPFLAGS += $(CPPFLAGS_NO_ARCH) $(ARCH_FLAGS) diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index 16175acca12..7b12421dc54 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -22,7 +22,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-C++' # TODO (mxyan): use version that match gRPC version when pod is stabilized - version = '1.31.0' + version = '1.31.1' s.version = version s.summary = 'gRPC C++ library' s.homepage = 'https://grpc.io' diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 545381bd520..0ab44ba8206 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-Core' - version = '1.31.0' + version = '1.31.1' s.version = version s.summary = 'Core cross-platform gRPC library, written in C' s.homepage = 'https://grpc.io' diff --git a/gRPC-ProtoRPC.podspec b/gRPC-ProtoRPC.podspec index 9065aa73111..14d51e6880a 100644 --- a/gRPC-ProtoRPC.podspec +++ b/gRPC-ProtoRPC.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-ProtoRPC' - version = '1.31.0' + version = '1.31.1' s.version = version s.summary = 'RPC library for Protocol Buffers, based on gRPC' s.homepage = 'https://grpc.io' diff --git a/gRPC-RxLibrary.podspec b/gRPC-RxLibrary.podspec index fdcd5082f31..0879c39b227 100644 --- a/gRPC-RxLibrary.podspec +++ b/gRPC-RxLibrary.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-RxLibrary' - version = '1.31.0' + version = '1.31.1' s.version = version s.summary = 'Reactive Extensions library for iOS/OSX.' s.homepage = 'https://grpc.io' diff --git a/gRPC.podspec b/gRPC.podspec index 74e65068233..1dc678b31ff 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -20,7 +20,7 @@ Pod::Spec.new do |s| s.name = 'gRPC' - version = '1.31.0' + version = '1.31.1' s.version = version s.summary = 'gRPC client library for iOS/OSX' s.homepage = 'https://grpc.io' diff --git a/package.xml b/package.xml index 2ca7237bac2..96f06404897 100644 --- a/package.xml +++ b/package.xml @@ -13,8 +13,8 @@ 2019-09-24 - 1.31.0 - 1.31.0 + 1.31.1 + 1.31.1 stable @@ -22,7 +22,7 @@ Apache 2.0 -- gRPC Core 1.31.0 update +- gRPC Core 1.31.1 update diff --git a/src/cpp/common/version_cc.cc b/src/cpp/common/version_cc.cc index 2106a7e12ac..af240283f7d 100644 --- a/src/cpp/common/version_cc.cc +++ b/src/cpp/common/version_cc.cc @@ -22,5 +22,5 @@ #include namespace grpc { -std::string Version() { return "1.31.0"; } +std::string Version() { return "1.31.1"; } } // namespace grpc diff --git a/src/csharp/Grpc.Core.Api/VersionInfo.cs b/src/csharp/Grpc.Core.Api/VersionInfo.cs index 7a094176e7a..30a6f07e419 100644 --- a/src/csharp/Grpc.Core.Api/VersionInfo.cs +++ b/src/csharp/Grpc.Core.Api/VersionInfo.cs @@ -33,11 +33,11 @@ namespace Grpc.Core /// /// Current AssemblyFileVersion of gRPC C# assemblies /// - public const string CurrentAssemblyFileVersion = "2.31.0.0"; + public const string CurrentAssemblyFileVersion = "2.31.1.0"; /// /// Current version of gRPC C# /// - public const string CurrentVersion = "2.31.0"; + public const string CurrentVersion = "2.31.1"; } } diff --git a/src/csharp/build/dependencies.props b/src/csharp/build/dependencies.props index de02dfbee69..7f6f0f79340 100644 --- a/src/csharp/build/dependencies.props +++ b/src/csharp/build/dependencies.props @@ -1,7 +1,7 @@ - 2.31.0 + 2.31.1 3.12.2 diff --git a/src/csharp/build_unitypackage.bat b/src/csharp/build_unitypackage.bat index 3384a91007d..fed939a95a7 100644 --- a/src/csharp/build_unitypackage.bat +++ b/src/csharp/build_unitypackage.bat @@ -13,7 +13,7 @@ @rem limitations under the License. @rem Current package versions -set VERSION=2.31.0 +set VERSION=2.31.1 @rem Adjust the location of nuget.exe set NUGET=C:\nuget\nuget.exe diff --git a/src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec index dcd2a8d7119..68e8dd22317 100644 --- a/src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec @@ -42,7 +42,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler-gRPCCppPlugin' - v = '1.31.0' + v = '1.31.1' s.version = v s.summary = 'The gRPC ProtoC plugin generates C++ files from .proto services.' s.description = <<-DESC diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index 793f9fb674e..5fad58467fa 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -42,7 +42,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler-gRPCPlugin' - v = '1.31.0' + v = '1.31.1' s.version = v s.summary = 'The gRPC ProtoC plugin generates Objective-C files from .proto services.' s.description = <<-DESC diff --git a/src/objective-c/GRPCClient/version.h b/src/objective-c/GRPCClient/version.h index 65d53ae047c..ee1a9050df3 100644 --- a/src/objective-c/GRPCClient/version.h +++ b/src/objective-c/GRPCClient/version.h @@ -22,4 +22,4 @@ // instead. This file can be regenerated from the template by running // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.31.0" +#define GRPC_OBJC_VERSION_STRING @"1.31.1" diff --git a/src/objective-c/tests/version.h b/src/objective-c/tests/version.h index 566310334f0..782a600037a 100644 --- a/src/objective-c/tests/version.h +++ b/src/objective-c/tests/version.h @@ -22,5 +22,5 @@ // instead. This file can be regenerated from the template by running // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.31.0" +#define GRPC_OBJC_VERSION_STRING @"1.31.1" #define GRPC_C_VERSION_STRING @"11.0.0" diff --git a/src/php/composer.json b/src/php/composer.json index 4a61a9a49e3..1c268855263 100644 --- a/src/php/composer.json +++ b/src/php/composer.json @@ -2,7 +2,7 @@ "name": "grpc/grpc-dev", "description": "gRPC library for PHP - for Development use only", "license": "Apache-2.0", - "version": "1.31.0", + "version": "1.31.1", "require": { "php": ">=5.5.0", "google/protobuf": "^v3.3.0" diff --git a/src/php/ext/grpc/version.h b/src/php/ext/grpc/version.h index c529fc01f2a..d08ec47141c 100644 --- a/src/php/ext/grpc/version.h +++ b/src/php/ext/grpc/version.h @@ -20,6 +20,6 @@ #ifndef VERSION_H #define VERSION_H -#define PHP_GRPC_VERSION "1.31.0" +#define PHP_GRPC_VERSION "1.31.1" #endif /* VERSION_H */ diff --git a/src/python/grpcio/grpc/_grpcio_metadata.py b/src/python/grpcio/grpc/_grpcio_metadata.py index 8b782192ec3..97678692aef 100644 --- a/src/python/grpcio/grpc/_grpcio_metadata.py +++ b/src/python/grpcio/grpc/_grpcio_metadata.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc/_grpcio_metadata.py.template`!!! -__version__ = """1.31.0""" +__version__ = """1.31.1""" diff --git a/src/python/grpcio/grpc_version.py b/src/python/grpcio/grpc_version.py index df98793910c..afeebded3ef 100644 --- a/src/python/grpcio/grpc_version.py +++ b/src/python/grpcio/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_version.py.template`!!! -VERSION = '1.31.0' +VERSION = '1.31.1' diff --git a/src/python/grpcio_channelz/grpc_version.py b/src/python/grpcio_channelz/grpc_version.py index 668ae3b3ff4..8e8a9f03a62 100644 --- a/src/python/grpcio_channelz/grpc_version.py +++ b/src/python/grpcio_channelz/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_channelz/grpc_version.py.template`!!! -VERSION = '1.31.0' +VERSION = '1.31.1' diff --git a/src/python/grpcio_health_checking/grpc_version.py b/src/python/grpcio_health_checking/grpc_version.py index d29cbb37145..9532584830f 100644 --- a/src/python/grpcio_health_checking/grpc_version.py +++ b/src/python/grpcio_health_checking/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_health_checking/grpc_version.py.template`!!! -VERSION = '1.31.0' +VERSION = '1.31.1' diff --git a/src/python/grpcio_reflection/grpc_version.py b/src/python/grpcio_reflection/grpc_version.py index 78e74f72f1c..9e9989076a9 100644 --- a/src/python/grpcio_reflection/grpc_version.py +++ b/src/python/grpcio_reflection/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_reflection/grpc_version.py.template`!!! -VERSION = '1.31.0' +VERSION = '1.31.1' diff --git a/src/python/grpcio_status/grpc_version.py b/src/python/grpcio_status/grpc_version.py index c7752ec7810..cdc41d48c96 100644 --- a/src/python/grpcio_status/grpc_version.py +++ b/src/python/grpcio_status/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_status/grpc_version.py.template`!!! -VERSION = '1.31.0' +VERSION = '1.31.1' diff --git a/src/python/grpcio_testing/grpc_version.py b/src/python/grpcio_testing/grpc_version.py index aa1e817fa11..dfc9dc89cae 100644 --- a/src/python/grpcio_testing/grpc_version.py +++ b/src/python/grpcio_testing/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_testing/grpc_version.py.template`!!! -VERSION = '1.31.0' +VERSION = '1.31.1' diff --git a/src/python/grpcio_tests/grpc_version.py b/src/python/grpcio_tests/grpc_version.py index 6fd3bee7a28..2f8cde9f0bd 100644 --- a/src/python/grpcio_tests/grpc_version.py +++ b/src/python/grpcio_tests/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_tests/grpc_version.py.template`!!! -VERSION = '1.31.0' +VERSION = '1.31.1' diff --git a/src/ruby/lib/grpc/version.rb b/src/ruby/lib/grpc/version.rb index 7eef11a7921..06c0e293ba6 100644 --- a/src/ruby/lib/grpc/version.rb +++ b/src/ruby/lib/grpc/version.rb @@ -14,5 +14,5 @@ # GRPC contains the General RPC module. module GRPC - VERSION = '1.31.0' + VERSION = '1.31.1' end diff --git a/src/ruby/tools/version.rb b/src/ruby/tools/version.rb index 4f01bb078cb..ef983461656 100644 --- a/src/ruby/tools/version.rb +++ b/src/ruby/tools/version.rb @@ -14,6 +14,6 @@ module GRPC module Tools - VERSION = '1.31.0' + VERSION = '1.31.1' end end diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index 4136af47eb9..cc8b71b0b0d 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION = '1.31.0' +VERSION = '1.31.1' diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 1310bc78f2c..dc10fc2e0b1 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.31.0 +PROJECT_NUMBER = 1.31.1 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index b2f2ea31749..6db2b116d14 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.31.0 +PROJECT_NUMBER = 1.31.1 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.objc b/tools/doxygen/Doxyfile.objc index b9f5b03d34b..568482861b5 100644 --- a/tools/doxygen/Doxyfile.objc +++ b/tools/doxygen/Doxyfile.objc @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC Objective-C" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.31.0 +PROJECT_NUMBER = 1.31.1 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.objc.internal b/tools/doxygen/Doxyfile.objc.internal index b14ee16d515..635095c9878 100644 --- a/tools/doxygen/Doxyfile.objc.internal +++ b/tools/doxygen/Doxyfile.objc.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC Objective-C" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.31.0 +PROJECT_NUMBER = 1.31.1 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.php b/tools/doxygen/Doxyfile.php index 58d2788f2f1..6d25c6dad06 100644 --- a/tools/doxygen/Doxyfile.php +++ b/tools/doxygen/Doxyfile.php @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC PHP" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.31.0 +PROJECT_NUMBER = 1.31.1 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a From 559115474385b08cb898f742eb744c2e3c44dde1 Mon Sep 17 00:00:00 2001 From: Hannah Shi Date: Tue, 11 Aug 2020 22:49:37 +0000 Subject: [PATCH 04/62] use absolute package name, add test cases --- src/compiler/ruby_generator_string-inl.h | 2 +- .../testing/same_package_service_name.proto | 27 +++++++++++++++++ .../same_ruby_package_service_name.proto | 29 +++++++++++++++++++ .../spec/pb/codegen/package_option_spec.rb | 20 +++++++++++++ 4 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 src/ruby/spec/pb/codegen/grpc/testing/same_package_service_name.proto create mode 100644 src/ruby/spec/pb/codegen/grpc/testing/same_ruby_package_service_name.proto diff --git a/src/compiler/ruby_generator_string-inl.h b/src/compiler/ruby_generator_string-inl.h index b3e19386c80..968f795cbb5 100644 --- a/src/compiler/ruby_generator_string-inl.h +++ b/src/compiler/ruby_generator_string-inl.h @@ -124,7 +124,7 @@ inline std::string RubyTypeOf(const grpc::protobuf::Descriptor* descriptor) { ReplacePrefix(&proto_type, ".", ""); // remove the leading . (no package) proto_type = RubyPackage(descriptor->file()) + "." + proto_type; } - std::string res(proto_type); + std::string res("." + proto_type); if (res.find('.') == std::string::npos) { return res; } else { diff --git a/src/ruby/spec/pb/codegen/grpc/testing/same_package_service_name.proto b/src/ruby/spec/pb/codegen/grpc/testing/same_package_service_name.proto new file mode 100644 index 00000000000..15dd8f0aec9 --- /dev/null +++ b/src/ruby/spec/pb/codegen/grpc/testing/same_package_service_name.proto @@ -0,0 +1,27 @@ +// Copyright 2020 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package same_name; + +service SameName { + rpc Health(Request) returns (Status); +} + +message Status { + string msg = 1; +} + +message Request {} diff --git a/src/ruby/spec/pb/codegen/grpc/testing/same_ruby_package_service_name.proto b/src/ruby/spec/pb/codegen/grpc/testing/same_ruby_package_service_name.proto new file mode 100644 index 00000000000..9ce631ca5b6 --- /dev/null +++ b/src/ruby/spec/pb/codegen/grpc/testing/same_ruby_package_service_name.proto @@ -0,0 +1,29 @@ +// Copyright 2020 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package other_name; + +option ruby_package = "SameName2"; + +service SameName2 { + rpc Health(Request) returns (Status); +} + +message Status { + string msg = 1; +} + +message Request {} diff --git a/src/ruby/spec/pb/codegen/package_option_spec.rb b/src/ruby/spec/pb/codegen/package_option_spec.rb index a4668ace520..f64b4f6aa62 100644 --- a/src/ruby/spec/pb/codegen/package_option_spec.rb +++ b/src/ruby/spec/pb/codegen/package_option_spec.rb @@ -48,6 +48,26 @@ describe 'Code Generation Options' do expect(services[:NestedMessageTest].output).to eq(RPC::Test::New::Package::Options::Bar::Baz) end end + + it 'should generate when package and service has same name' do + with_protos(['grpc/testing/same_package_service_name.proto']) do + expect { SameName::SameName::Service }.to raise_error(NameError) + expect(require('grpc/testing/same_package_service_name_services_pb')).to be_truthy + expect { SameName::SameName::Service }.to_not raise_error + expect { SameName::Request }.to_not raise_error + expect { SameName::Status }.to_not raise_error + end + end + + it 'should generate when ruby_package and service has same name' do + with_protos(['grpc/testing/same_ruby_package_service_name.proto']) do + expect { SameName2::SameName2::Service }.to raise_error(NameError) + expect(require('grpc/testing/same_ruby_package_service_name_services_pb')).to be_truthy + expect { SameName2::SameName2::Service }.to_not raise_error + expect { SameName2::Request }.to_not raise_error + expect { SameName2::Status }.to_not raise_error + end + end end def with_protos(file_paths) From 3541005ee19275c8cf3605b489041e66aaea9aac Mon Sep 17 00:00:00 2001 From: Hannah Shi Date: Wed, 12 Aug 2020 05:36:00 +0000 Subject: [PATCH 05/62] change test case to absolute package name --- src/ruby/end2end/package_with_underscore_checker.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ruby/end2end/package_with_underscore_checker.rb b/src/ruby/end2end/package_with_underscore_checker.rb index 27ea00ffa7a..043e621d847 100644 --- a/src/ruby/end2end/package_with_underscore_checker.rb +++ b/src/ruby/end2end/package_with_underscore_checker.rb @@ -43,8 +43,8 @@ def main end correct_modularized_rpc = 'rpc :TestOne, ' \ - 'Grpc::Testing::PackageWithUnderscore::Data::Request, ' \ - 'Grpc::Testing::PackageWithUnderscore::Data::Response' + '::Grpc::Testing::PackageWithUnderscore::Data::Request, ' \ + '::Grpc::Testing::PackageWithUnderscore::Data::Response' return if got.include?(correct_modularized_rpc) From 0894f21c2fa7cc281bc584c0e08cbbbad1d04b28 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Thu, 13 Aug 2020 23:00:32 -0700 Subject: [PATCH 06/62] Ran generate_proto_ruby.sh to update generated files --- src/ruby/bin/math_services_pb.rb | 8 ++-- .../pb/grpc/health/v1/health_services_pb.rb | 4 +- .../pb/src/proto/grpc/testing/messages_pb.rb | 5 +++ .../proto/grpc/testing/test_services_pb.rb | 40 +++++++++++++------ .../testing/benchmark_service_services_pb.rb | 10 ++--- .../qps/src/proto/grpc/testing/messages_pb.rb | 5 +++ ...report_qps_scenario_service_services_pb.rb | 2 +- .../testing/worker_service_services_pb.rb | 8 ++-- 8 files changed, 54 insertions(+), 28 deletions(-) diff --git a/src/ruby/bin/math_services_pb.rb b/src/ruby/bin/math_services_pb.rb index e6f32e3d01a..961117ccfaf 100644 --- a/src/ruby/bin/math_services_pb.rb +++ b/src/ruby/bin/math_services_pb.rb @@ -31,19 +31,19 @@ module Math # Div divides DivArgs.dividend by DivArgs.divisor and returns the quotient # and remainder. - rpc :Div, DivArgs, DivReply + rpc :Div, ::Math::DivArgs, ::Math::DivReply # DivMany accepts an arbitrary number of division args from the client stream # and sends back the results in the reply stream. The stream continues until # the client closes its end; the server does the same after sending all the # replies. The stream ends immediately if either end aborts. - rpc :DivMany, stream(DivArgs), stream(DivReply) + rpc :DivMany, stream(::Math::DivArgs), stream(::Math::DivReply) # Fib generates numbers in the Fibonacci sequence. If FibArgs.limit > 0, Fib # generates up to limit numbers; otherwise it continues until the call is # canceled. Unlike Fib above, Fib has no final FibReply. - rpc :Fib, FibArgs, stream(Num) + rpc :Fib, ::Math::FibArgs, stream(::Math::Num) # Sum sums a stream of numbers, returning the final result once the stream # is closed. - rpc :Sum, stream(Num), Num + rpc :Sum, stream(::Math::Num), ::Math::Num end Stub = Service.rpc_stub_class diff --git a/src/ruby/pb/grpc/health/v1/health_services_pb.rb b/src/ruby/pb/grpc/health/v1/health_services_pb.rb index 5992f1c403d..351e7e150a5 100644 --- a/src/ruby/pb/grpc/health/v1/health_services_pb.rb +++ b/src/ruby/pb/grpc/health/v1/health_services_pb.rb @@ -36,7 +36,7 @@ module Grpc # If the requested service is unknown, the call will fail with status # NOT_FOUND. - rpc :Check, HealthCheckRequest, HealthCheckResponse + rpc :Check, ::Grpc::Health::V1::HealthCheckRequest, ::Grpc::Health::V1::HealthCheckResponse # Performs a watch for the serving status of the requested service. # The server will immediately send back a message indicating the current # serving status. It will then subsequently send a new message whenever @@ -52,7 +52,7 @@ module Grpc # should assume this method is not supported and should not retry the # call. If the call terminates with any other status (including OK), # clients should retry the call with appropriate exponential backoff. - rpc :Watch, HealthCheckRequest, stream(HealthCheckResponse) + rpc :Watch, ::Grpc::Health::V1::HealthCheckRequest, stream(::Grpc::Health::V1::HealthCheckResponse) end Stub = Service.rpc_stub_class diff --git a/src/ruby/pb/src/proto/grpc/testing/messages_pb.rb b/src/ruby/pb/src/proto/grpc/testing/messages_pb.rb index f492ccfa670..d902ae0e736 100644 --- a/src/ruby/pb/src/proto/grpc/testing/messages_pb.rb +++ b/src/ruby/pb/src/proto/grpc/testing/messages_pb.rb @@ -71,6 +71,10 @@ Google::Protobuf::DescriptorPool.generated_pool.build do add_message "grpc.testing.LoadBalancerStatsResponse" do map :rpcs_by_peer, :string, :int32, 1 optional :num_failures, :int32, 2 + map :rpcs_by_method, :string, :message, 3, "grpc.testing.LoadBalancerStatsResponse.RpcsByPeer" + end + add_message "grpc.testing.LoadBalancerStatsResponse.RpcsByPeer" do + map :rpcs_by_peer, :string, :int32, 1 end add_enum "grpc.testing.PayloadType" do value :COMPRESSABLE, 0 @@ -99,6 +103,7 @@ module Grpc ReconnectInfo = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.testing.ReconnectInfo").msgclass LoadBalancerStatsRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.testing.LoadBalancerStatsRequest").msgclass LoadBalancerStatsResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.testing.LoadBalancerStatsResponse").msgclass + LoadBalancerStatsResponse::RpcsByPeer = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.testing.LoadBalancerStatsResponse.RpcsByPeer").msgclass PayloadType = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.testing.PayloadType").enummodule GrpclbRouteType = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.testing.GrpclbRouteType").enummodule end diff --git a/src/ruby/pb/src/proto/grpc/testing/test_services_pb.rb b/src/ruby/pb/src/proto/grpc/testing/test_services_pb.rb index 8138bd0114d..115acd0dcae 100644 --- a/src/ruby/pb/src/proto/grpc/testing/test_services_pb.rb +++ b/src/ruby/pb/src/proto/grpc/testing/test_services_pb.rb @@ -36,31 +36,31 @@ module Grpc self.service_name = 'grpc.testing.TestService' # One empty request followed by one empty response. - rpc :EmptyCall, Empty, Empty + rpc :EmptyCall, ::Grpc::Testing::Empty, ::Grpc::Testing::Empty # One request followed by one response. - rpc :UnaryCall, SimpleRequest, SimpleResponse + rpc :UnaryCall, ::Grpc::Testing::SimpleRequest, ::Grpc::Testing::SimpleResponse # One request followed by one response. Response has cache control # headers set such that a caching HTTP proxy (such as GFE) can # satisfy subsequent requests. - rpc :CacheableUnaryCall, SimpleRequest, SimpleResponse + rpc :CacheableUnaryCall, ::Grpc::Testing::SimpleRequest, ::Grpc::Testing::SimpleResponse # One request followed by a sequence of responses (streamed download). # The server returns the payload with client desired type and sizes. - rpc :StreamingOutputCall, StreamingOutputCallRequest, stream(StreamingOutputCallResponse) + rpc :StreamingOutputCall, ::Grpc::Testing::StreamingOutputCallRequest, stream(::Grpc::Testing::StreamingOutputCallResponse) # A sequence of requests followed by one response (streamed upload). # The server returns the aggregated size of client payload as the result. - rpc :StreamingInputCall, stream(StreamingInputCallRequest), StreamingInputCallResponse + rpc :StreamingInputCall, stream(::Grpc::Testing::StreamingInputCallRequest), ::Grpc::Testing::StreamingInputCallResponse # A sequence of requests with each request served by the server immediately. # As one request could lead to multiple responses, this interface # demonstrates the idea of full duplexing. - rpc :FullDuplexCall, stream(StreamingOutputCallRequest), stream(StreamingOutputCallResponse) + rpc :FullDuplexCall, stream(::Grpc::Testing::StreamingOutputCallRequest), stream(::Grpc::Testing::StreamingOutputCallResponse) # A sequence of requests followed by a sequence of responses. # The server buffers all the client requests and then serves them in order. A # stream of responses are returned to the client when the server starts with # first request. - rpc :HalfDuplexCall, stream(StreamingOutputCallRequest), stream(StreamingOutputCallResponse) + rpc :HalfDuplexCall, stream(::Grpc::Testing::StreamingOutputCallRequest), stream(::Grpc::Testing::StreamingOutputCallResponse) # The test server will not implement this method. It will be used # to test the behavior when clients call unimplemented methods. - rpc :UnimplementedCall, Empty, Empty + rpc :UnimplementedCall, ::Grpc::Testing::Empty, ::Grpc::Testing::Empty end Stub = Service.rpc_stub_class @@ -77,7 +77,7 @@ module Grpc self.service_name = 'grpc.testing.UnimplementedService' # A call that no server should implement - rpc :UnimplementedCall, Empty, Empty + rpc :UnimplementedCall, ::Grpc::Testing::Empty, ::Grpc::Testing::Empty end Stub = Service.rpc_stub_class @@ -92,8 +92,8 @@ module Grpc self.unmarshal_class_method = :decode self.service_name = 'grpc.testing.ReconnectService' - rpc :Start, ReconnectParams, Empty - rpc :Stop, Empty, ReconnectInfo + rpc :Start, ::Grpc::Testing::ReconnectParams, ::Grpc::Testing::Empty + rpc :Stop, ::Grpc::Testing::Empty, ::Grpc::Testing::ReconnectInfo end Stub = Service.rpc_stub_class @@ -109,7 +109,23 @@ module Grpc self.service_name = 'grpc.testing.LoadBalancerStatsService' # Gets the backend distribution for RPCs sent by a test client. - rpc :GetClientStats, LoadBalancerStatsRequest, LoadBalancerStatsResponse + rpc :GetClientStats, ::Grpc::Testing::LoadBalancerStatsRequest, ::Grpc::Testing::LoadBalancerStatsResponse + end + + Stub = Service.rpc_stub_class + end + module XdsUpdateHealthService + # A service to remotely control health status of an xDS test server. + class Service + + include GRPC::GenericService + + self.marshal_class_method = :encode + self.unmarshal_class_method = :decode + self.service_name = 'grpc.testing.XdsUpdateHealthService' + + rpc :SetServing, ::Grpc::Testing::Empty, ::Grpc::Testing::Empty + rpc :SetNotServing, ::Grpc::Testing::Empty, ::Grpc::Testing::Empty end Stub = Service.rpc_stub_class diff --git a/src/ruby/qps/src/proto/grpc/testing/benchmark_service_services_pb.rb b/src/ruby/qps/src/proto/grpc/testing/benchmark_service_services_pb.rb index 65e5a75c4d8..63e2d5d20fa 100644 --- a/src/ruby/qps/src/proto/grpc/testing/benchmark_service_services_pb.rb +++ b/src/ruby/qps/src/proto/grpc/testing/benchmark_service_services_pb.rb @@ -34,20 +34,20 @@ module Grpc # One request followed by one response. # The server returns the client payload as-is. - rpc :UnaryCall, SimpleRequest, SimpleResponse + rpc :UnaryCall, ::Grpc::Testing::SimpleRequest, ::Grpc::Testing::SimpleResponse # Repeated sequence of one request followed by one response. # Should be called streaming ping-pong # The server returns the client payload as-is on each response - rpc :StreamingCall, stream(SimpleRequest), stream(SimpleResponse) + rpc :StreamingCall, stream(::Grpc::Testing::SimpleRequest), stream(::Grpc::Testing::SimpleResponse) # Single-sided unbounded streaming from client to server # The server returns the client payload as-is once the client does WritesDone - rpc :StreamingFromClient, stream(SimpleRequest), SimpleResponse + rpc :StreamingFromClient, stream(::Grpc::Testing::SimpleRequest), ::Grpc::Testing::SimpleResponse # Single-sided unbounded streaming from server to client # The server repeatedly returns the client payload as-is - rpc :StreamingFromServer, SimpleRequest, stream(SimpleResponse) + rpc :StreamingFromServer, ::Grpc::Testing::SimpleRequest, stream(::Grpc::Testing::SimpleResponse) # Two-sided unbounded streaming between server to client # Both sides send the content of their own choice to the other - rpc :StreamingBothWays, stream(SimpleRequest), stream(SimpleResponse) + rpc :StreamingBothWays, stream(::Grpc::Testing::SimpleRequest), stream(::Grpc::Testing::SimpleResponse) end Stub = Service.rpc_stub_class diff --git a/src/ruby/qps/src/proto/grpc/testing/messages_pb.rb b/src/ruby/qps/src/proto/grpc/testing/messages_pb.rb index f492ccfa670..d902ae0e736 100644 --- a/src/ruby/qps/src/proto/grpc/testing/messages_pb.rb +++ b/src/ruby/qps/src/proto/grpc/testing/messages_pb.rb @@ -71,6 +71,10 @@ Google::Protobuf::DescriptorPool.generated_pool.build do add_message "grpc.testing.LoadBalancerStatsResponse" do map :rpcs_by_peer, :string, :int32, 1 optional :num_failures, :int32, 2 + map :rpcs_by_method, :string, :message, 3, "grpc.testing.LoadBalancerStatsResponse.RpcsByPeer" + end + add_message "grpc.testing.LoadBalancerStatsResponse.RpcsByPeer" do + map :rpcs_by_peer, :string, :int32, 1 end add_enum "grpc.testing.PayloadType" do value :COMPRESSABLE, 0 @@ -99,6 +103,7 @@ module Grpc ReconnectInfo = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.testing.ReconnectInfo").msgclass LoadBalancerStatsRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.testing.LoadBalancerStatsRequest").msgclass LoadBalancerStatsResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.testing.LoadBalancerStatsResponse").msgclass + LoadBalancerStatsResponse::RpcsByPeer = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.testing.LoadBalancerStatsResponse.RpcsByPeer").msgclass PayloadType = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.testing.PayloadType").enummodule GrpclbRouteType = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.testing.GrpclbRouteType").enummodule end diff --git a/src/ruby/qps/src/proto/grpc/testing/report_qps_scenario_service_services_pb.rb b/src/ruby/qps/src/proto/grpc/testing/report_qps_scenario_service_services_pb.rb index ddc81bed3da..5e41cfeead1 100644 --- a/src/ruby/qps/src/proto/grpc/testing/report_qps_scenario_service_services_pb.rb +++ b/src/ruby/qps/src/proto/grpc/testing/report_qps_scenario_service_services_pb.rb @@ -33,7 +33,7 @@ module Grpc self.service_name = 'grpc.testing.ReportQpsScenarioService' # Report results of a QPS test benchmark scenario. - rpc :ReportScenario, ScenarioResult, Void + rpc :ReportScenario, ::Grpc::Testing::ScenarioResult, ::Grpc::Testing::Void end Stub = Service.rpc_stub_class diff --git a/src/ruby/qps/src/proto/grpc/testing/worker_service_services_pb.rb b/src/ruby/qps/src/proto/grpc/testing/worker_service_services_pb.rb index a7ecc957571..049db477789 100644 --- a/src/ruby/qps/src/proto/grpc/testing/worker_service_services_pb.rb +++ b/src/ruby/qps/src/proto/grpc/testing/worker_service_services_pb.rb @@ -38,18 +38,18 @@ module Grpc # stats. Closing the stream will initiate shutdown of the test server # and once the shutdown has finished, the OK status is sent to terminate # this RPC. - rpc :RunServer, stream(ServerArgs), stream(ServerStatus) + rpc :RunServer, stream(::Grpc::Testing::ServerArgs), stream(::Grpc::Testing::ServerStatus) # Start client with specified workload. # First request sent specifies the ClientConfig followed by ClientStatus # response. After that, a "Mark" can be sent anytime to request the latest # stats. Closing the stream will initiate shutdown of the test client # and once the shutdown has finished, the OK status is sent to terminate # this RPC. - rpc :RunClient, stream(ClientArgs), stream(ClientStatus) + rpc :RunClient, stream(::Grpc::Testing::ClientArgs), stream(::Grpc::Testing::ClientStatus) # Just return the core count - unary call - rpc :CoreCount, CoreRequest, CoreResponse + rpc :CoreCount, ::Grpc::Testing::CoreRequest, ::Grpc::Testing::CoreResponse # Quit this worker - rpc :QuitWorker, Void, Void + rpc :QuitWorker, ::Grpc::Testing::Void, ::Grpc::Testing::Void end Stub = Service.rpc_stub_class From a095e3ee7a3426cd29e2c484e739463d8f0233c3 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Sat, 8 Aug 2020 00:17:51 +0000 Subject: [PATCH 07/62] Update XDS interop tests for PHP and Ruby --- .../Src/Proto/Grpc/Testing/Messages.php | 2 +- .../Src/Proto/Grpc/Testing/Test.php | 2 +- .../Testing/LoadBalancerStatsResponse.php | 27 +++ .../LoadBalancerStatsResponse/RpcsByPeer.php | 68 ++++++++ .../LoadBalancerStatsResponse_RpcsByPeer.php | 16 ++ .../Testing/XdsUpdateHealthServiceClient.php | 66 ++++++++ src/php/tests/interop/xds_client.php | 159 +++++++++++++++--- src/ruby/pb/test/xds_client.rb | 118 +++++++++++-- .../linux/grpc_xds_php_test_in_docker.sh | 6 +- .../linux/grpc_xds_ruby_test_in_docker.sh | 6 +- 10 files changed, 420 insertions(+), 50 deletions(-) create mode 100644 src/php/tests/interop/Grpc/Testing/LoadBalancerStatsResponse/RpcsByPeer.php create mode 100644 src/php/tests/interop/Grpc/Testing/LoadBalancerStatsResponse_RpcsByPeer.php create mode 100644 src/php/tests/interop/Grpc/Testing/XdsUpdateHealthServiceClient.php diff --git a/src/php/tests/interop/GPBMetadata/Src/Proto/Grpc/Testing/Messages.php b/src/php/tests/interop/GPBMetadata/Src/Proto/Grpc/Testing/Messages.php index 91a9c7bfcf1..199ccf40d56 100644 --- a/src/php/tests/interop/GPBMetadata/Src/Proto/Grpc/Testing/Messages.php +++ b/src/php/tests/interop/GPBMetadata/Src/Proto/Grpc/Testing/Messages.php @@ -15,7 +15,7 @@ class Messages return; } $pool->internalAddGeneratedFile(hex2bin( - "0ad70e0a257372632f70726f746f2f677270632f74657374696e672f6d657373616765732e70726f746f120c677270632e74657374696e67221a0a09426f6f6c56616c7565120d0a0576616c756518012001280822400a075061796c6f616412270a047479706518012001280e32192e677270632e74657374696e672e5061796c6f616454797065120c0a04626f647918022001280c222b0a0a4563686f537461747573120c0a04636f6465180120012805120f0a076d6573736167651802200128092286030a0d53696d706c655265717565737412300a0d726573706f6e73655f7479706518012001280e32192e677270632e74657374696e672e5061796c6f61645479706512150a0d726573706f6e73655f73697a6518022001280512260a077061796c6f616418032001280b32152e677270632e74657374696e672e5061796c6f616412150a0d66696c6c5f757365726e616d6518042001280812180a1066696c6c5f6f617574685f73636f706518052001280812340a13726573706f6e73655f636f6d7072657373656418062001280b32172e677270632e74657374696e672e426f6f6c56616c756512310a0f726573706f6e73655f73746174757318072001280b32182e677270632e74657374696e672e4563686f53746174757312320a116578706563745f636f6d7072657373656418082001280b32172e677270632e74657374696e672e426f6f6c56616c756512160a0e66696c6c5f7365727665725f6964180920012808121e0a1666696c6c5f677270636c625f726f7574655f74797065180a2001280822be010a0e53696d706c65526573706f6e736512260a077061796c6f616418012001280b32152e677270632e74657374696e672e5061796c6f616412100a08757365726e616d6518022001280912130a0b6f617574685f73636f706518032001280912110a097365727665725f696418042001280912380a11677270636c625f726f7574655f7479706518052001280e321d2e677270632e74657374696e672e477270636c62526f7574655479706512100a08686f73746e616d6518062001280922770a1953747265616d696e67496e70757443616c6c5265717565737412260a077061796c6f616418012001280b32152e677270632e74657374696e672e5061796c6f616412320a116578706563745f636f6d7072657373656418022001280b32172e677270632e74657374696e672e426f6f6c56616c7565223d0a1a53747265616d696e67496e70757443616c6c526573706f6e7365121f0a17616767726567617465645f7061796c6f61645f73697a6518012001280522640a12526573706f6e7365506172616d6574657273120c0a0473697a6518012001280512130a0b696e74657276616c5f7573180220012805122b0a0a636f6d7072657373656418032001280b32172e677270632e74657374696e672e426f6f6c56616c756522e8010a1a53747265616d696e674f757470757443616c6c5265717565737412300a0d726573706f6e73655f7479706518012001280e32192e677270632e74657374696e672e5061796c6f616454797065123d0a13726573706f6e73655f706172616d657465727318022003280b32202e677270632e74657374696e672e526573706f6e7365506172616d657465727312260a077061796c6f616418032001280b32152e677270632e74657374696e672e5061796c6f616412310a0f726573706f6e73655f73746174757318072001280b32182e677270632e74657374696e672e4563686f53746174757322450a1b53747265616d696e674f757470757443616c6c526573706f6e736512260a077061796c6f616418012001280b32152e677270632e74657374696e672e5061796c6f616422330a0f5265636f6e6e656374506172616d7312200a186d61785f7265636f6e6e6563745f6261636b6f66665f6d7318012001280522330a0d5265636f6e6e656374496e666f120e0a0670617373656418012001280812120a0a6261636b6f66665f6d7318022003280522410a184c6f616442616c616e63657253746174735265717565737412100a086e756d5f7270637318012001280512130a0b74696d656f75745f73656318022001280522b3010a194c6f616442616c616e6365725374617473526573706f6e7365124d0a0c727063735f62795f7065657218012003280b32372e677270632e74657374696e672e4c6f616442616c616e6365725374617473526573706f6e73652e52706373427950656572456e74727912140a0c6e756d5f6661696c757265731802200128051a310a0f52706373427950656572456e747279120b0a036b6579180120012809120d0a0576616c75651802200128053a0238012a1f0a0b5061796c6f61645479706512100a0c434f4d505245535341424c4510002a6f0a0f477270636c62526f75746554797065121d0a19475250434c425f524f5554455f545950455f554e4b4e4f574e1000121e0a1a475250434c425f524f5554455f545950455f46414c4c4241434b1001121d0a19475250434c425f524f5554455f545950455f4241434b454e441002620670726f746f33" + "0aaf110a257372632f70726f746f2f677270632f74657374696e672f6d657373616765732e70726f746f120c677270632e74657374696e67221a0a09426f6f6c56616c7565120d0a0576616c756518012001280822400a075061796c6f616412270a047479706518012001280e32192e677270632e74657374696e672e5061796c6f616454797065120c0a04626f647918022001280c222b0a0a4563686f537461747573120c0a04636f6465180120012805120f0a076d6573736167651802200128092286030a0d53696d706c655265717565737412300a0d726573706f6e73655f7479706518012001280e32192e677270632e74657374696e672e5061796c6f61645479706512150a0d726573706f6e73655f73697a6518022001280512260a077061796c6f616418032001280b32152e677270632e74657374696e672e5061796c6f616412150a0d66696c6c5f757365726e616d6518042001280812180a1066696c6c5f6f617574685f73636f706518052001280812340a13726573706f6e73655f636f6d7072657373656418062001280b32172e677270632e74657374696e672e426f6f6c56616c756512310a0f726573706f6e73655f73746174757318072001280b32182e677270632e74657374696e672e4563686f53746174757312320a116578706563745f636f6d7072657373656418082001280b32172e677270632e74657374696e672e426f6f6c56616c756512160a0e66696c6c5f7365727665725f6964180920012808121e0a1666696c6c5f677270636c625f726f7574655f74797065180a2001280822be010a0e53696d706c65526573706f6e736512260a077061796c6f616418012001280b32152e677270632e74657374696e672e5061796c6f616412100a08757365726e616d6518022001280912130a0b6f617574685f73636f706518032001280912110a097365727665725f696418042001280912380a11677270636c625f726f7574655f7479706518052001280e321d2e677270632e74657374696e672e477270636c62526f7574655479706512100a08686f73746e616d6518062001280922770a1953747265616d696e67496e70757443616c6c5265717565737412260a077061796c6f616418012001280b32152e677270632e74657374696e672e5061796c6f616412320a116578706563745f636f6d7072657373656418022001280b32172e677270632e74657374696e672e426f6f6c56616c7565223d0a1a53747265616d696e67496e70757443616c6c526573706f6e7365121f0a17616767726567617465645f7061796c6f61645f73697a6518012001280522640a12526573706f6e7365506172616d6574657273120c0a0473697a6518012001280512130a0b696e74657276616c5f7573180220012805122b0a0a636f6d7072657373656418032001280b32172e677270632e74657374696e672e426f6f6c56616c756522e8010a1a53747265616d696e674f757470757443616c6c5265717565737412300a0d726573706f6e73655f7479706518012001280e32192e677270632e74657374696e672e5061796c6f616454797065123d0a13726573706f6e73655f706172616d657465727318022003280b32202e677270632e74657374696e672e526573706f6e7365506172616d657465727312260a077061796c6f616418032001280b32152e677270632e74657374696e672e5061796c6f616412310a0f726573706f6e73655f73746174757318072001280b32182e677270632e74657374696e672e4563686f53746174757322450a1b53747265616d696e674f757470757443616c6c526573706f6e736512260a077061796c6f616418012001280b32152e677270632e74657374696e672e5061796c6f616422330a0f5265636f6e6e656374506172616d7312200a186d61785f7265636f6e6e6563745f6261636b6f66665f6d7318012001280522330a0d5265636f6e6e656374496e666f120e0a0670617373656418012001280812120a0a6261636b6f66665f6d7318022003280522410a184c6f616442616c616e63657253746174735265717565737412100a086e756d5f7270637318012001280512130a0b74696d656f75745f736563180220012805228b040a194c6f616442616c616e6365725374617473526573706f6e7365124d0a0c727063735f62795f7065657218012003280b32372e677270632e74657374696e672e4c6f616442616c616e6365725374617473526573706f6e73652e52706373427950656572456e74727912140a0c6e756d5f6661696c7572657318022001280512510a0e727063735f62795f6d6574686f6418032003280b32392e677270632e74657374696e672e4c6f616442616c616e6365725374617473526573706f6e73652e5270637342794d6574686f64456e7472791a99010a0a5270637342795065657212580a0c727063735f62795f7065657218012003280b32422e677270632e74657374696e672e4c6f616442616c616e6365725374617473526573706f6e73652e527063734279506565722e52706373427950656572456e7472791a310a0f52706373427950656572456e747279120b0a036b6579180120012809120d0a0576616c75651802200128053a0238011a310a0f52706373427950656572456e747279120b0a036b6579180120012809120d0a0576616c75651802200128053a0238011a670a115270637342794d6574686f64456e747279120b0a036b657918012001280912410a0576616c756518022001280b32322e677270632e74657374696e672e4c6f616442616c616e6365725374617473526573706f6e73652e527063734279506565723a0238012a1f0a0b5061796c6f61645479706512100a0c434f4d505245535341424c4510002a6f0a0f477270636c62526f75746554797065121d0a19475250434c425f524f5554455f545950455f554e4b4e4f574e1000121e0a1a475250434c425f524f5554455f545950455f46414c4c4241434b1001121d0a19475250434c425f524f5554455f545950455f4241434b454e441002620670726f746f33" ), true); static::$is_initialized = true; diff --git a/src/php/tests/interop/GPBMetadata/Src/Proto/Grpc/Testing/Test.php b/src/php/tests/interop/GPBMetadata/Src/Proto/Grpc/Testing/Test.php index 2ae2dadabdc..d950b30a38e 100644 --- a/src/php/tests/interop/GPBMetadata/Src/Proto/Grpc/Testing/Test.php +++ b/src/php/tests/interop/GPBMetadata/Src/Proto/Grpc/Testing/Test.php @@ -17,7 +17,7 @@ class Test \GPBMetadata\Src\Proto\Grpc\Testing\PBEmpty::initOnce(); \GPBMetadata\Src\Proto\Grpc\Testing\Messages::initOnce(); $pool->internalAddGeneratedFile(hex2bin( - "0aca090a217372632f70726f746f2f677270632f74657374696e672f746573742e70726f746f120c677270632e74657374696e671a257372632f70726f746f2f677270632f74657374696e672f6d657373616765732e70726f746f32e7050a0b546573745365727669636512430a09456d70747943616c6c121a2e677270632e74657374696e672e456d7074794d6573736167651a1a2e677270632e74657374696e672e456d7074794d65737361676512460a09556e61727943616c6c121b2e677270632e74657374696e672e53696d706c65526571756573741a1c2e677270632e74657374696e672e53696d706c65526573706f6e7365124f0a12436163686561626c65556e61727943616c6c121b2e677270632e74657374696e672e53696d706c65526571756573741a1c2e677270632e74657374696e672e53696d706c65526573706f6e7365126c0a1353747265616d696e674f757470757443616c6c12282e677270632e74657374696e672e53747265616d696e674f757470757443616c6c526571756573741a292e677270632e74657374696e672e53747265616d696e674f757470757443616c6c526573706f6e7365300112690a1253747265616d696e67496e70757443616c6c12272e677270632e74657374696e672e53747265616d696e67496e70757443616c6c526571756573741a282e677270632e74657374696e672e53747265616d696e67496e70757443616c6c526573706f6e7365280112690a0e46756c6c4475706c657843616c6c12282e677270632e74657374696e672e53747265616d696e674f757470757443616c6c526571756573741a292e677270632e74657374696e672e53747265616d696e674f757470757443616c6c526573706f6e73652801300112690a0e48616c664475706c657843616c6c12282e677270632e74657374696e672e53747265616d696e674f757470757443616c6c526571756573741a292e677270632e74657374696e672e53747265616d696e674f757470757443616c6c526573706f6e736528013001124b0a11556e696d706c656d656e74656443616c6c121a2e677270632e74657374696e672e456d7074794d6573736167651a1a2e677270632e74657374696e672e456d7074794d65737361676532630a14556e696d706c656d656e74656453657276696365124b0a11556e696d706c656d656e74656443616c6c121a2e677270632e74657374696e672e456d7074794d6573736167651a1a2e677270632e74657374696e672e456d7074794d6573736167653297010a105265636f6e6e6563745365727669636512420a055374617274121d2e677270632e74657374696e672e5265636f6e6e656374506172616d731a1a2e677270632e74657374696e672e456d7074794d657373616765123f0a0453746f70121a2e677270632e74657374696e672e456d7074794d6573736167651a1b2e677270632e74657374696e672e5265636f6e6e656374496e666f327f0a184c6f616442616c616e63657253746174735365727669636512630a0e476574436c69656e74537461747312262e677270632e74657374696e672e4c6f616442616c616e6365725374617473526571756573741a272e677270632e74657374696e672e4c6f616442616c616e6365725374617473526573706f6e73652200620670726f746f33" + "0af40a0a217372632f70726f746f2f677270632f74657374696e672f746573742e70726f746f120c677270632e74657374696e671a257372632f70726f746f2f677270632f74657374696e672f6d657373616765732e70726f746f32e7050a0b546573745365727669636512430a09456d70747943616c6c121a2e677270632e74657374696e672e456d7074794d6573736167651a1a2e677270632e74657374696e672e456d7074794d65737361676512460a09556e61727943616c6c121b2e677270632e74657374696e672e53696d706c65526571756573741a1c2e677270632e74657374696e672e53696d706c65526573706f6e7365124f0a12436163686561626c65556e61727943616c6c121b2e677270632e74657374696e672e53696d706c65526571756573741a1c2e677270632e74657374696e672e53696d706c65526573706f6e7365126c0a1353747265616d696e674f757470757443616c6c12282e677270632e74657374696e672e53747265616d696e674f757470757443616c6c526571756573741a292e677270632e74657374696e672e53747265616d696e674f757470757443616c6c526573706f6e7365300112690a1253747265616d696e67496e70757443616c6c12272e677270632e74657374696e672e53747265616d696e67496e70757443616c6c526571756573741a282e677270632e74657374696e672e53747265616d696e67496e70757443616c6c526573706f6e7365280112690a0e46756c6c4475706c657843616c6c12282e677270632e74657374696e672e53747265616d696e674f757470757443616c6c526571756573741a292e677270632e74657374696e672e53747265616d696e674f757470757443616c6c526573706f6e73652801300112690a0e48616c664475706c657843616c6c12282e677270632e74657374696e672e53747265616d696e674f757470757443616c6c526571756573741a292e677270632e74657374696e672e53747265616d696e674f757470757443616c6c526573706f6e736528013001124b0a11556e696d706c656d656e74656443616c6c121a2e677270632e74657374696e672e456d7074794d6573736167651a1a2e677270632e74657374696e672e456d7074794d65737361676532630a14556e696d706c656d656e74656453657276696365124b0a11556e696d706c656d656e74656443616c6c121a2e677270632e74657374696e672e456d7074794d6573736167651a1a2e677270632e74657374696e672e456d7074794d6573736167653297010a105265636f6e6e6563745365727669636512420a055374617274121d2e677270632e74657374696e672e5265636f6e6e656374506172616d731a1a2e677270632e74657374696e672e456d7074794d657373616765123f0a0453746f70121a2e677270632e74657374696e672e456d7074794d6573736167651a1b2e677270632e74657374696e672e5265636f6e6e656374496e666f327f0a184c6f616442616c616e63657253746174735365727669636512630a0e476574436c69656e74537461747312262e677270632e74657374696e672e4c6f616442616c616e6365725374617473526571756573741a272e677270632e74657374696e672e4c6f616442616c616e6365725374617473526573706f6e7365220032a7010a165864735570646174654865616c74685365727669636512440a0a53657453657276696e67121a2e677270632e74657374696e672e456d7074794d6573736167651a1a2e677270632e74657374696e672e456d7074794d65737361676512470a0d5365744e6f7453657276696e67121a2e677270632e74657374696e672e456d7074794d6573736167651a1a2e677270632e74657374696e672e456d7074794d657373616765620670726f746f33" ), true); static::$is_initialized = true; diff --git a/src/php/tests/interop/Grpc/Testing/LoadBalancerStatsResponse.php b/src/php/tests/interop/Grpc/Testing/LoadBalancerStatsResponse.php index 270189e1871..e1188b5e417 100644 --- a/src/php/tests/interop/Grpc/Testing/LoadBalancerStatsResponse.php +++ b/src/php/tests/interop/Grpc/Testing/LoadBalancerStatsResponse.php @@ -25,6 +25,10 @@ class LoadBalancerStatsResponse extends \Google\Protobuf\Internal\Message * Generated from protobuf field int32 num_failures = 2; */ protected $num_failures = 0; + /** + * Generated from protobuf field map rpcs_by_method = 3; + */ + private $rpcs_by_method; /** * Constructor. @@ -36,6 +40,7 @@ class LoadBalancerStatsResponse extends \Google\Protobuf\Internal\Message * The number of completed RPCs for each peer. * @type int $num_failures * The number of RPCs that failed to record a remote peer. + * @type array|\Google\Protobuf\Internal\MapField $rpcs_by_method * } */ public function __construct($data = NULL) { @@ -95,5 +100,27 @@ class LoadBalancerStatsResponse extends \Google\Protobuf\Internal\Message return $this; } + /** + * Generated from protobuf field map rpcs_by_method = 3; + * @return \Google\Protobuf\Internal\MapField + */ + public function getRpcsByMethod() + { + return $this->rpcs_by_method; + } + + /** + * Generated from protobuf field map rpcs_by_method = 3; + * @param array|\Google\Protobuf\Internal\MapField $var + * @return $this + */ + public function setRpcsByMethod($var) + { + $arr = GPBUtil::checkMapField($var, \Google\Protobuf\Internal\GPBType::STRING, \Google\Protobuf\Internal\GPBType::MESSAGE, \Grpc\Testing\LoadBalancerStatsResponse\RpcsByPeer::class); + $this->rpcs_by_method = $arr; + + return $this; + } + } diff --git a/src/php/tests/interop/Grpc/Testing/LoadBalancerStatsResponse/RpcsByPeer.php b/src/php/tests/interop/Grpc/Testing/LoadBalancerStatsResponse/RpcsByPeer.php new file mode 100644 index 00000000000..07515c472e5 --- /dev/null +++ b/src/php/tests/interop/Grpc/Testing/LoadBalancerStatsResponse/RpcsByPeer.php @@ -0,0 +1,68 @@ +grpc.testing.LoadBalancerStatsResponse.RpcsByPeer + */ +class RpcsByPeer extends \Google\Protobuf\Internal\Message +{ + /** + * The number of completed RPCs for each peer. + * + * Generated from protobuf field map rpcs_by_peer = 1; + */ + private $rpcs_by_peer; + + /** + * Constructor. + * + * @param array $data { + * Optional. Data for populating the Message object. + * + * @type array|\Google\Protobuf\Internal\MapField $rpcs_by_peer + * The number of completed RPCs for each peer. + * } + */ + public function __construct($data = NULL) { + \GPBMetadata\Src\Proto\Grpc\Testing\Messages::initOnce(); + parent::__construct($data); + } + + /** + * The number of completed RPCs for each peer. + * + * Generated from protobuf field map rpcs_by_peer = 1; + * @return \Google\Protobuf\Internal\MapField + */ + public function getRpcsByPeer() + { + return $this->rpcs_by_peer; + } + + /** + * The number of completed RPCs for each peer. + * + * Generated from protobuf field map rpcs_by_peer = 1; + * @param array|\Google\Protobuf\Internal\MapField $var + * @return $this + */ + public function setRpcsByPeer($var) + { + $arr = GPBUtil::checkMapField($var, \Google\Protobuf\Internal\GPBType::STRING, \Google\Protobuf\Internal\GPBType::INT32); + $this->rpcs_by_peer = $arr; + + return $this; + } + +} + +// Adding a class alias for backwards compatibility with the previous class name. +class_alias(RpcsByPeer::class, \Grpc\Testing\LoadBalancerStatsResponse_RpcsByPeer::class); + diff --git a/src/php/tests/interop/Grpc/Testing/LoadBalancerStatsResponse_RpcsByPeer.php b/src/php/tests/interop/Grpc/Testing/LoadBalancerStatsResponse_RpcsByPeer.php new file mode 100644 index 00000000000..d7c35a7f076 --- /dev/null +++ b/src/php/tests/interop/Grpc/Testing/LoadBalancerStatsResponse_RpcsByPeer.php @@ -0,0 +1,16 @@ +_simpleRequest('/grpc.testing.XdsUpdateHealthService/SetServing', + $argument, + ['\Grpc\Testing\EmptyMessage', 'decode'], + $metadata, $options); + } + + /** + * @param \Grpc\Testing\EmptyMessage $argument input argument + * @param array $metadata metadata + * @param array $options call options + * @return \Grpc\Testing\EmptyMessage + */ + public function SetNotServing(\Grpc\Testing\EmptyMessage $argument, + $metadata = [], $options = []) { + return $this->_simpleRequest('/grpc.testing.XdsUpdateHealthService/SetNotServing', + $argument, + ['\Grpc\Testing\EmptyMessage', 'decode'], + $metadata, $options); + } + +} diff --git a/src/php/tests/interop/xds_client.php b/src/php/tests/interop/xds_client.php index 6e967d2a1f6..6927ce30a7d 100644 --- a/src/php/tests/interop/xds_client.php +++ b/src/php/tests/interop/xds_client.php @@ -33,13 +33,14 @@ class LoadBalancerStatsService function getClientStats(\Grpc\Testing\LoadBalancerStatsRequest $request) { $num_rpcs = $request->getNumRpcs(); $timeout_sec = $request->getTimeoutSec(); + $rpcs_by_method = []; $rpcs_by_peer = []; - $num_failures = $num_rpcs; + $num_failures = 0; // Heavy limitation now: the server is blocking, until all // the necessary num_rpcs are finished, or timeout is reached global $client_thread; - $start_id = count($client_thread->results) + 1; + $start_id = $client_thread->num_results + 1; $end_id = $start_id + $num_rpcs; $now = hrtime(true); $timeout = $now[0] + ($now[1] / 1e9) + $timeout_sec; @@ -50,7 +51,7 @@ class LoadBalancerStatsService break; } // Thread variable seems to be read-only - $curr_id = count($client_thread->results); + $curr_id = $client_thread->num_results; if ($curr_id >= $end_id) { break; } @@ -58,19 +59,52 @@ class LoadBalancerStatsService } // Tally up results - $end_id = min($end_id, count($client_thread->results)); - for ($i = $start_id; $i < $end_id; $i++) { - $hostname = $client_thread->results[$i]; - if ($hostname) { - $num_failures -= 1; - if (!array_key_exists($hostname, $rpcs_by_peer)) { - $rpcs_by_peer[$hostname] = 0; + $end_id = min($end_id, $client_thread->num_results); + // "$client_thread->results" will be in the form of + // [ + // 'rpc1' => [ + // 'hostname1', '', 'hostname2', 'hostname1', '', ... + // ], + // 'rpc2' => [ + // '', 'hostname1', 'hostname2', '', 'hostname2', ... + // ], + // ] + foreach ($client_thread->results as $rpc => $results) { + // initialize, can always start from scratch here + $rpcs_by_method[$rpc] = []; + for ($i = $start_id; $i < $end_id; $i++) { + $hostname = $results[$i]; + if ($hostname) { + // initialize in case we haven't seen this hostname + // before + if (!array_key_exists($hostname, $rpcs_by_method[$rpc])) { + $rpcs_by_method[$rpc][$hostname] = 0; + } + if (!array_key_exists($hostname, $rpcs_by_peer)) { + $rpcs_by_peer[$hostname] = 0; + } + // increment the remote hostname distribution histogram + // both by overall, and broken down per RPC + $rpcs_by_method[$rpc][$hostname] += 1; + $rpcs_by_peer[$hostname] += 1; + } else { + // $num_failures here are counted per individual RPC + $num_failures += 1; } - $rpcs_by_peer[$hostname] += 1; } } + + // Convert our hashmaps above into protobuf objects $response = new Grpc\Testing\LoadBalancerStatsResponse(); + $rpcs_by_method_map = []; + foreach ($rpcs_by_method as $rpc => $rpcs_by_peer_per_method) { + $rpcs_by_peer_proto_obj + = new Grpc\Testing\LoadBalancerStatsResponse\RpcsByPeer(); + $rpcs_by_peer_proto_obj->setRpcsByPeer($rpcs_by_peer_per_method); + $rpcs_by_method_map[$rpc] = $rpcs_by_peer_proto_obj; + } $response->setRpcsByPeer($rpcs_by_peer); + $response->setRpcsByMethod($rpcs_by_method_map); $response->setNumFailures($num_failures); return $response; } @@ -83,28 +117,74 @@ class ClientThread extends Thread { private $target_seconds_between_rpcs_; private $fail_on_failed_rpcs_; private $autoload_path_; + private $TIMEOUT_US = 30 * 1e6; // 30 seconds + public $num_results = 0; public $results; - + public function __construct($server_address, $qps, $fail_on_failed_rpcs, + $rpcs_to_send, $metadata_to_send, $autoload_path) { $this->server_address_ = $server_address; $this->target_seconds_between_rpcs_ = 1.0 / $qps; $this->fail_on_failed_rpcs_ = $fail_on_failed_rpcs; + $this->rpcs_to_send = explode(',', $rpcs_to_send); + // Convert input in the form of + // rpc1:k1:v1,rpc2:k2:v2,rpc1:k3:v3 + // into + // [ + // 'rpc1' => [ + // 'k1' => 'v1', + // 'k3' => 'v3', + // ], + // 'rpc2' => [ + // 'k2' => 'v2' + // ], + // ] + $this->metadata_to_send = []; + if ($_all_metadata = explode(',', $metadata_to_send)) { + foreach ($_all_metadata as $one_metadata_pair) { + list($rpc, + $metadata_key, + $metadata_value) = explode(':', $one_metadata_pair); + // initialize in case we haven't seen this rpc before + if (!array_key_exists($rpc, $this->metadata_to_send)) { + $this->metadata_to_send[$rpc] = []; + } + $this->metadata_to_send[$rpc][$metadata_key] + = $metadata_value; + } + } $this->autoload_path_ = $autoload_path; + $this->simple_request = new Grpc\Testing\SimpleRequest(); + $this->empty_request = new Grpc\Testing\EmptyMessage(); $this->results = []; + foreach ($this->rpcs_to_send as $rpc) { + $this->results[$rpc] = []; + } + } + + public function sendUnaryCall($stub, $metadata) { + return $stub->UnaryCall($this->simple_request, + $metadata, + ['timeout' => $this->TIMEOUT_US]); + } + + public function sendEmptyCall($stub, $metadata) { + return $stub->EmptyCall($this->empty_request, + $metadata, + ['timeout' => $this->TIMEOUT_US]); } public function run() { // Autoloaded classes do not get inherited in threads. // Hence we need to do this. require_once($this->autoload_path_); - $TIMEOUT_US = 30 * 1e6; // 30 seconds $stub = new Grpc\Testing\TestServiceClient($this->server_address_, [ 'credentials' => Grpc\ChannelCredentials::createInsecure() ]); - $request = new Grpc\Testing\SimpleRequest(); - $target_next_start_us = hrtime(true) / 1000; # hrtime returns nanoseconds + # hrtime returns nanoseconds + $target_next_start_us = hrtime(true) / 1000; while (true) { $now_us = hrtime(true) / 1000; $sleep_us = $target_next_start_us - $now_us; @@ -121,18 +201,43 @@ class ClientThread extends Thread { ($this->target_seconds_between_rpcs_ * 1e6); usleep($sleep_us); } - list($response, $status) - = $stub->UnaryCall($request, [], - ['timeout' => $TIMEOUT_US])->wait(); - if ($status->code == Grpc\STATUS_OK) { - $this->results[] = $response->getHostname(); - } else { - if ($this->fail_on_failed_rpcs_) { - throw new Exception('UnaryCall failed with status ' - . $status->code); + foreach ($this->rpcs_to_send as $rpc) { + $metadata = array_key_exists( + $rpc, $this->metadata_to_send) ? + $this->metadata_to_send[$rpc] : []; + // This copy is somehow necessary because + // $this->metadata_to_send[$rpc] somehow becomes a + // Volatile object, instead of an associative array. + $metadata_array = []; + foreach ($metadata as $key => $value) { + $metadata_array[$key] = [$value]; + } + $call = null; + if ($rpc == 'UnaryCall') { + $call = $this->sendUnaryCall($stub, $metadata_array); + } else if ($rpc == 'EmptyCall') { + $call = $this->sendEmptyCall($stub, $metadata_array); + } else { + throw new Exception("Unhandled rpc $rpc"); + } + // the remote peer is being returned as part of the + // initial metadata, according to the test spec + $initial_metadata = $call->getMetadata(); + list($response, $status) = $call->wait(); + if ($status->code == Grpc\STATUS_OK && + array_key_exists('hostname', $initial_metadata)) { + $this->results[$rpc][] = $initial_metadata['hostname'][0]; + } else { + if ($this->fail_on_failed_rpcs_) { + throw new Exception("$rpc failed with status " + . $status->code); + } + $this->results[$rpc][] = ""; } - $this->results[] = ""; } + // $num_results here is only incremented when the group of + // all $rpcs_to_send are done. + $this->num_results++; } } @@ -145,10 +250,14 @@ class ClientThread extends Thread { // Note: num_channels are currently ignored for now $args = getopt('', ['fail_on_failed_rpcs:', 'num_channels:', + 'rpc:', 'metadata:', 'server:', 'stats_port:', 'qps:']); $client_thread = new ClientThread($args['server'], $args['qps'], $args['fail_on_failed_rpcs'], + (empty($args['rpc']) ? 'UnaryCall' + : $args['rpc']), + $args['metadata'], $autoload_path); $client_thread->start(); diff --git a/src/ruby/pb/test/xds_client.rb b/src/ruby/pb/test/xds_client.rb index a70095ccd1f..54bf27df299 100755 --- a/src/ruby/pb/test/xds_client.rb +++ b/src/ruby/pb/test/xds_client.rb @@ -81,6 +81,7 @@ class TestTarget < Grpc::Testing::LoadBalancerStatsService::Service watcher = {} $watchers_mutex.synchronize do watcher = { + "rpcs_by_method" => Hash.new(), "rpcs_by_peer" => Hash.new(0), "rpcs_needed" => req['num_rpcs'], "no_remote_peer" => 0 @@ -95,17 +96,45 @@ class TestTarget < Grpc::Testing::LoadBalancerStatsService::Service end $watchers.delete_at($watchers.index(watcher)) end + # convert results into proper proto object + rpcs_by_method = {} + watcher['rpcs_by_method'].each do | rpc_name, rpcs_by_peer | + rpcs_by_method[rpc_name] = LoadBalancerStatsResponse::RpcsByPeer.new( + rpcs_by_peer: rpcs_by_peer + ) + end LoadBalancerStatsResponse.new( + rpcs_by_method: rpcs_by_method, rpcs_by_peer: watcher['rpcs_by_peer'], num_failures: watcher['no_remote_peer'] + watcher['rpcs_needed'] ); end end +# execute 1 RPC and return remote hostname +def execute_rpc(op, fail_on_failed_rpcs) + remote_peer = "" + begin + op.execute + if op.metadata.key?('hostname') + remote_peer = op.metadata['hostname'] + end + rescue GRPC::BadStatus => e + GRPC.logger.info("ruby xds: rpc failed:|#{e.message}|, " \ + "this may or may not be expected") + if fail_on_failed_rpcs + raise e + end + end + remote_peer +end + # send 1 rpc every 1/qps second -def run_test_loop(stub, target_seconds_between_rpcs, fail_on_failed_rpcs) +def run_test_loop(stub, target_seconds_between_rpcs, fail_on_failed_rpcs, + rpcs_to_send, metadata_to_send) include Grpc::Testing - req = SimpleRequest.new() + simple_req = SimpleRequest.new() + empty_req = Empty.new() target_next_start = Process.clock_gettime(Process::CLOCK_MONOTONIC) while !$shutdown now = Process.clock_gettime(Process::CLOCK_MONOTONIC) @@ -121,25 +150,42 @@ def run_test_loop(stub, target_seconds_between_rpcs, fail_on_failed_rpcs) target_next_start += target_seconds_between_rpcs sleep(sleep_seconds) end - begin - deadline = GRPC::Core::TimeConsts::from_relative_time(30) # 30 seconds - resp = stub.unary_call(req, deadline: deadline) - remote_peer = resp.hostname - rescue GRPC::BadStatus => e - remote_peer = "" - GRPC.logger.info("ruby xds: rpc failed:|#{e.message}|, " \ - "this may or may not be expected") - if fail_on_failed_rpcs - raise e + deadline = GRPC::Core::TimeConsts::from_relative_time(30) # 30 seconds + results = {} + rpcs_to_send.each do |rpc| + metadata = metadata_to_send.key?(rpc) ? metadata_to_send[rpc] : {} + if rpc == 'UnaryCall' + op = stub.unary_call(simple_req, + metadata: metadata, + deadline: deadline, + return_op: true) + elsif rpc == 'EmptyCall' + op = stub.empty_call(empty_req, + metadata: metadata, + deadline: deadline, + return_op: true) + else + raise "Unsupported rpc %s" % [rpc] end + results[rpc] = execute_rpc(op, fail_on_failed_rpcs) end $watchers_mutex.synchronize do $watchers.each do |watcher| + # this is counted once when each group of all rpcs_to_send were done watcher['rpcs_needed'] -= 1 - if remote_peer.strip.empty? - watcher['no_remote_peer'] += 1 - else - watcher['rpcs_by_peer'][remote_peer] += 1 + results.each do | rpc_name, remote_peer | + if remote_peer.strip.empty? + # error is counted per individual RPC + watcher['no_remote_peer'] += 1 + else + if not watcher['rpcs_by_method'].key?(rpc_name) + watcher['rpcs_by_method'][rpc_name] = Hash.new(0) + end + # increment the remote hostname distribution histogram + # both by overall, and broken down per RPC + watcher['rpcs_by_method'][rpc_name][remote_peer] += 1 + watcher['rpcs_by_peer'][remote_peer] += 1 + end end end $watchers_cv.broadcast @@ -149,6 +195,7 @@ end # Args is used to hold the command line info. Args = Struct.new(:fail_on_failed_rpcs, :num_channels, + :rpc, :metadata, :server, :stats_port, :qps) # validates the command line options, returning them as a Hash. @@ -156,6 +203,8 @@ def parse_args args = Args.new args['fail_on_failed_rpcs'] = false args['num_channels'] = 1 + args['rpc'] = 'UnaryCall' + args['metadata'] = '' OptionParser.new do |opts| opts.on('--fail_on_failed_rpcs BOOL', ['false', 'true']) do |v| args['fail_on_failed_rpcs'] = v == 'true' @@ -163,6 +212,12 @@ def parse_args opts.on('--num_channels CHANNELS', 'number of channels') do |v| args['num_channels'] = v.to_i end + opts.on('--rpc RPCS_TO_SEND', 'list of RPCs to send') do |v| + args['rpc'] = v + end + opts.on('--metadata METADATA_TO_SEND', 'metadata to send per RPC') do |v| + args['metadata'] = v + end opts.on('--server SERVER_HOST', 'server hostname') do |v| GRPC.logger.info("ruby xds: server address is #{v}") args['server'] = v @@ -195,11 +250,40 @@ def main # The client just sends unary rpcs continuously in a regular interval stub = create_stub(opts) target_seconds_between_rpcs = (1.0 / opts['qps'].to_f) + rpcs_to_send = [] + if opts['rpc'] + rpcs_to_send = opts['rpc'].split(',') + end + # Convert 'metadata' input in the form of + # rpc1:k1:v1,rpc2:k2:v2,rpc1:k3:v3 + # into + # { + # 'rpc1' => { + # 'k1' => 'v1', + # 'k3' => 'v3', + # }, + # 'rpc2' => { + # 'k2' => 'v2' + # }, + # } + metadata_to_send = {} + if opts['metadata'] + metadata_entries = opts['metadata'].split(',') + metadata_entries.each do |e| + (rpc_name, metadata_key, metadata_value) = e.split(':') + # initialize if we haven't seen this rpc_name yet + if !metadata_to_send.key?(rpc_name) + metadata_to_send[rpc_name] = {} + end + metadata_to_send[rpc_name][metadata_key] = metadata_value + end + end client_threads = Array.new opts['num_channels'].times { client_threads << Thread.new { run_test_loop(stub, target_seconds_between_rpcs, - opts['fail_on_failed_rpcs']) + opts['fail_on_failed_rpcs'], + rpcs_to_send, metadata_to_send) } } diff --git a/tools/internal_ci/linux/grpc_xds_php_test_in_docker.sh b/tools/internal_ci/linux/grpc_xds_php_test_in_docker.sh index 0d1deffcf5e..2ff670fd7a1 100755 --- a/tools/internal_ci/linux/grpc_xds_php_test_in_docker.sh +++ b/tools/internal_ci/linux/grpc_xds_php_test_in_docker.sh @@ -60,10 +60,10 @@ export CC=/usr/bin/gcc GRPC_VERBOSITY=debug GRPC_TRACE=xds_client,xds_resolver,xds_routing_lb,cds_lb,eds_lb,priority_lb,weighted_target_lb,lrs_lb "$PYTHON" \ tools/run_tests/run_xds_tests.py \ - --test_case=all \ + --test_case="all,path_matching,header_matching" \ --project_id=grpc-testing \ - --source_image=projects/grpc-testing/global/images/xds-test-server \ + --source_image=projects/grpc-testing/global/images/xds-test-server-2 \ --path_to_server_binary=/java_server/grpc-java/interop-testing/build/install/grpc-interop-testing/bin/xds-test-server \ --gcp_suffix=$(date '+%s') \ --verbose \ - --client_cmd='php -d extension=grpc.so -d extension=pthreads.so src/php/tests/interop/xds_client.php --server=xds:///{server_uri} --stats_port={stats_port} --qps={qps}' + --client_cmd='php -d extension=grpc.so -d extension=pthreads.so src/php/tests/interop/xds_client.php --server=xds:///{server_uri} --stats_port={stats_port} --qps={qps} {fail_on_failed_rpc} {rpcs_to_send} {metadata_to_send}' diff --git a/tools/internal_ci/linux/grpc_xds_ruby_test_in_docker.sh b/tools/internal_ci/linux/grpc_xds_ruby_test_in_docker.sh index c88f6e3e5f3..de2ab4b58cd 100644 --- a/tools/internal_ci/linux/grpc_xds_ruby_test_in_docker.sh +++ b/tools/internal_ci/linux/grpc_xds_ruby_test_in_docker.sh @@ -50,10 +50,10 @@ touch "$TOOLS_DIR"/src/proto/grpc/testing/__init__.py GRPC_VERBOSITY=debug GRPC_TRACE=xds_client,xds_resolver,xds_routing_lb,cds_lb,eds_lb,priority_lb,weighted_target_lb,lrs_lb "$PYTHON" \ tools/run_tests/run_xds_tests.py \ - --test_case=all \ + --test_case="all,path_matching,header_matching" \ --project_id=grpc-testing \ - --source_image=projects/grpc-testing/global/images/xds-test-server \ + --source_image=projects/grpc-testing/global/images/xds-test-server-2 \ --path_to_server_binary=/java_server/grpc-java/interop-testing/build/install/grpc-interop-testing/bin/xds-test-server \ --gcp_suffix=$(date '+%s') \ --verbose \ - --client_cmd='ruby src/ruby/pb/test/xds_client.rb --server=xds:///{server_uri} --stats_port={stats_port} --qps={qps}' + --client_cmd='ruby src/ruby/pb/test/xds_client.rb --server=xds:///{server_uri} --stats_port={stats_port} --qps={qps} {fail_on_failed_rpc} {rpcs_to_send} {metadata_to_send}' From 2cd4c4b52294a0cead0d42646e91bc3c40f86671 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Mon, 17 Aug 2020 18:45:12 +0000 Subject: [PATCH 08/62] Review fixes --- src/ruby/pb/test/xds_client.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ruby/pb/test/xds_client.rb b/src/ruby/pb/test/xds_client.rb index 54bf27df299..2f15c5fa522 100755 --- a/src/ruby/pb/test/xds_client.rb +++ b/src/ruby/pb/test/xds_client.rb @@ -98,7 +98,7 @@ class TestTarget < Grpc::Testing::LoadBalancerStatsService::Service end # convert results into proper proto object rpcs_by_method = {} - watcher['rpcs_by_method'].each do | rpc_name, rpcs_by_peer | + watcher['rpcs_by_method'].each do |rpc_name, rpcs_by_peer| rpcs_by_method[rpc_name] = LoadBalancerStatsResponse::RpcsByPeer.new( rpcs_by_peer: rpcs_by_peer ) @@ -173,7 +173,7 @@ def run_test_loop(stub, target_seconds_between_rpcs, fail_on_failed_rpcs, $watchers.each do |watcher| # this is counted once when each group of all rpcs_to_send were done watcher['rpcs_needed'] -= 1 - results.each do | rpc_name, remote_peer | + results.each do |rpc_name, remote_peer| if remote_peer.strip.empty? # error is counted per individual RPC watcher['no_remote_peer'] += 1 From 73d9be0c65024070d26386f3e04ecb72dedc949c Mon Sep 17 00:00:00 2001 From: Menghan Li Date: Tue, 11 Aug 2020 14:46:07 -0700 Subject: [PATCH 09/62] xds testing: add a test case with multiple routes referring to same cluster --- tools/run_tests/run_xds_tests.py | 36 ++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/tools/run_tests/run_xds_tests.py b/tools/run_tests/run_xds_tests.py index a5d6fa5c454..6f02a0daa1e 100755 --- a/tools/run_tests/run_xds_tests.py +++ b/tools/run_tests/run_xds_tests.py @@ -828,12 +828,40 @@ def test_path_matching(gcp, original_backend_service, instance_group, { "UnaryCall": alternate_backend_instances, "EmptyCall": original_backend_instances + }), + ( + # This test case is similar to the one above (but with route + # services swapped). This test has two routes (full_path and + # the default) to match EmptyCall, and both routes set + # alternative_backend_service as the action. This forces the + # client to handle duplicate Clusters in the RDS response. + [ + { + 'priority': 0, + # Prefix UnaryCall -> original_backend_service. + 'matchRules': [{ + 'prefixMatch': '/grpc.testing.TestService/Unary' + }], + 'service': original_backend_service.url + }, + { + 'priority': 1, + # FullPath EmptyCall -> alternate_backend_service. + 'matchRules': [{ + 'fullPathMatch': + '/grpc.testing.TestService/EmptyCall' + }], + 'service': alternate_backend_service.url + } + ], + { + "UnaryCall": original_backend_instances, + "EmptyCall": alternate_backend_instances }) ] for (route_rules, expected_instances) in test_cases: - logger.info('patching url map with %s -> alternative', - route_rules[0]['matchRules']) + logger.info('patching url map with %s', route_rules) patch_url_map_backend_service(gcp, original_backend_service, route_rules=route_rules) @@ -846,8 +874,8 @@ def test_path_matching(gcp, original_backend_service, instance_group, original_backend_instances + alternate_backend_instances, _WAIT_FOR_STATS_SEC) - retry_count = 10 - # Each attempt takes about 10 seconds, 10 retries is equivalent to 100 + retry_count = 20 + # Each attempt takes about 10 seconds, 20 retries is equivalent to 200 # seconds timeout. for i in range(retry_count): stats = get_client_stats(_NUM_TEST_RPCS, _WAIT_FOR_STATS_SEC) From 967c0a2a32d90b01f2682b5b54b20feb50dfa6db Mon Sep 17 00:00:00 2001 From: Karthik Ravi Shankar Date: Thu, 20 Aug 2020 14:51:40 -0700 Subject: [PATCH 10/62] Move Server from ::grpc_impl to ::grpc Reverts: https://github.com/grpc/grpc/pull/18458 --- BUILD | 1 - BUILD.gn | 1 - CMakeLists.txt | 2 - Makefile | 2 - build_autogenerated.yaml | 2 - gRPC-C++.podspec | 1 - .../impl/codegen/async_generic_service.h | 12 +- .../grpcpp/impl/codegen/async_stream_impl.h | 2 +- .../grpcpp/impl/codegen/completion_queue.h | 12 +- .../grpcpp/impl/codegen/server_context_impl.h | 6 +- include/grpcpp/impl/codegen/service_type.h | 2 +- include/grpcpp/security/server_credentials.h | 7 +- include/grpcpp/server.h | 354 +++++++++++++++- include/grpcpp/server_impl.h | 387 ------------------ src/cpp/server/server_cc.cc | 6 +- tools/doxygen/Doxyfile.c++ | 1 - tools/doxygen/Doxyfile.c++.internal | 1 - 17 files changed, 371 insertions(+), 428 deletions(-) delete mode 100644 include/grpcpp/server_impl.h diff --git a/BUILD b/BUILD index 5ec511c504e..4862fc6cafd 100644 --- a/BUILD +++ b/BUILD @@ -251,7 +251,6 @@ GRPCXX_PUBLIC_HDRS = [ "include/grpcpp/security/server_credentials.h", "include/grpcpp/security/tls_credentials_options.h", "include/grpcpp/server.h", - "include/grpcpp/server_impl.h", "include/grpcpp/server_builder.h", "include/grpcpp/server_context.h", "include/grpcpp/server_posix.h", diff --git a/BUILD.gn b/BUILD.gn index 56d77fc6700..f1bbedcc40c 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -1196,7 +1196,6 @@ config("grpc_config") { "include/grpcpp/server.h", "include/grpcpp/server_builder.h", "include/grpcpp/server_context.h", - "include/grpcpp/server_impl.h", "include/grpcpp/server_posix.h", "include/grpcpp/support/async_stream.h", "include/grpcpp/support/async_stream_impl.h", diff --git a/CMakeLists.txt b/CMakeLists.txt index 3b63d02f1e3..15bb131f1ee 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2857,7 +2857,6 @@ foreach(_hdr include/grpcpp/server.h include/grpcpp/server_builder.h include/grpcpp/server_context.h - include/grpcpp/server_impl.h include/grpcpp/server_posix.h include/grpcpp/support/async_stream.h include/grpcpp/support/async_stream_impl.h @@ -3542,7 +3541,6 @@ foreach(_hdr include/grpcpp/server.h include/grpcpp/server_builder.h include/grpcpp/server_context.h - include/grpcpp/server_impl.h include/grpcpp/server_posix.h include/grpcpp/support/async_stream.h include/grpcpp/support/async_stream_impl.h diff --git a/Makefile b/Makefile index 9fc06b0edf3..b17607f9706 100644 --- a/Makefile +++ b/Makefile @@ -3178,7 +3178,6 @@ PUBLIC_HEADERS_CXX += \ include/grpcpp/server.h \ include/grpcpp/server_builder.h \ include/grpcpp/server_context.h \ - include/grpcpp/server_impl.h \ include/grpcpp/server_posix.h \ include/grpcpp/support/async_stream.h \ include/grpcpp/support/async_stream_impl.h \ @@ -3708,7 +3707,6 @@ PUBLIC_HEADERS_CXX += \ include/grpcpp/server.h \ include/grpcpp/server_builder.h \ include/grpcpp/server_context.h \ - include/grpcpp/server_impl.h \ include/grpcpp/server_posix.h \ include/grpcpp/support/async_stream.h \ include/grpcpp/support/async_stream_impl.h \ diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index f62b8017c15..972de902e90 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -2189,7 +2189,6 @@ libs: - include/grpcpp/server.h - include/grpcpp/server_builder.h - include/grpcpp/server_context.h - - include/grpcpp/server_impl.h - include/grpcpp/server_posix.h - include/grpcpp/support/async_stream.h - include/grpcpp/support/async_stream_impl.h @@ -2566,7 +2565,6 @@ libs: - include/grpcpp/server.h - include/grpcpp/server_builder.h - include/grpcpp/server_context.h - - include/grpcpp/server_impl.h - include/grpcpp/server_posix.h - include/grpcpp/support/async_stream.h - include/grpcpp/support/async_stream_impl.h diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index 92aeba7b020..f58003d63a2 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -164,7 +164,6 @@ Pod::Spec.new do |s| 'include/grpcpp/server.h', 'include/grpcpp/server_builder.h', 'include/grpcpp/server_context.h', - 'include/grpcpp/server_impl.h', 'include/grpcpp/server_posix.h', 'include/grpcpp/support/async_stream.h', 'include/grpcpp/support/async_stream_impl.h', diff --git a/include/grpcpp/impl/codegen/async_generic_service.h b/include/grpcpp/impl/codegen/async_generic_service.h index 3cb20fbfc33..8415c2cb458 100644 --- a/include/grpcpp/impl/codegen/async_generic_service.h +++ b/include/grpcpp/impl/codegen/async_generic_service.h @@ -44,7 +44,7 @@ class GenericServerContext final : public ::grpc_impl::ServerContext { const std::string& host() const { return host_; } private: - friend class grpc_impl::Server; + friend class grpc::Server; friend class grpc::ServerInterface; void Clear() { @@ -84,8 +84,8 @@ class AsyncGenericService final { ::grpc::ServerCompletionQueue* notification_cq, void* tag); private: - friend class grpc_impl::Server; - grpc_impl::Server* server_; + friend class grpc::Server; + grpc::Server* server_; }; #ifndef GRPC_CALLBACK_API_NONEXPERIMENTAL @@ -105,7 +105,7 @@ class GenericCallbackServerContext final const std::string& host() const { return host_; } private: - friend class ::grpc_impl::Server; + friend class ::grpc::Server; friend class ::grpc::ServerInterface; void Clear() { @@ -140,7 +140,7 @@ class CallbackGenericService { } private: - friend class ::grpc_impl::Server; + friend class grpc::Server; ::grpc_impl::internal::CallbackBidiHandler* Handler() { @@ -151,7 +151,7 @@ class CallbackGenericService { }); } - grpc_impl::Server* server_{nullptr}; + grpc::Server* server_{nullptr}; }; #ifndef GRPC_CALLBACK_API_NONEXPERIMENTAL diff --git a/include/grpcpp/impl/codegen/async_stream_impl.h b/include/grpcpp/impl/codegen/async_stream_impl.h index 13952d2eb98..fc978dc5a08 100644 --- a/include/grpcpp/impl/codegen/async_stream_impl.h +++ b/include/grpcpp/impl/codegen/async_stream_impl.h @@ -1097,7 +1097,7 @@ class ServerAsyncReaderWriter final } private: - friend class ::grpc_impl::Server; + friend class ::grpc::Server; void BindCall(::grpc::internal::Call* call) override { call_ = *call; } diff --git a/include/grpcpp/impl/codegen/completion_queue.h b/include/grpcpp/impl/codegen/completion_queue.h index f2012460371..1dd9bc9e2e4 100644 --- a/include/grpcpp/impl/codegen/completion_queue.h +++ b/include/grpcpp/impl/codegen/completion_queue.h @@ -46,7 +46,6 @@ struct grpc_completion_queue; namespace grpc_impl { -class Server; template class ClientReader; template @@ -78,6 +77,7 @@ namespace grpc { class Channel; class ChannelInterface; +class Server; class ServerBuilder; class ServerInterface; @@ -252,7 +252,7 @@ class CompletionQueue : private ::grpc::GrpcLibraryCodegen { // Friends for access to server registration lists that enable checking and // logging on shutdown friend class ::grpc::ServerBuilder; - friend class ::grpc_impl::Server; + friend class ::grpc::Server; // Friend synchronous wrappers so that they can access Pluck(), which is // a semi-private API geared towards the synchronous implementation. @@ -382,14 +382,14 @@ class CompletionQueue : private ::grpc::GrpcLibraryCodegen { } } - void RegisterServer(const ::grpc_impl::Server* server) { + void RegisterServer(const ::grpc::Server* server) { (void)server; #ifndef NDEBUG grpc::internal::MutexLock l(&server_list_mutex_); server_list_.push_back(server); #endif } - void UnregisterServer(const ::grpc_impl::Server* server) { + void UnregisterServer(const ::grpc::Server* server) { (void)server; #ifndef NDEBUG grpc::internal::MutexLock l(&server_list_mutex_); @@ -412,7 +412,7 @@ class CompletionQueue : private ::grpc::GrpcLibraryCodegen { // NDEBUG, instantiate it in all cases since otherwise the size will be // inconsistent. mutable grpc::internal::Mutex server_list_mutex_; - std::list + std::list server_list_ /* GUARDED_BY(server_list_mutex_) */; }; @@ -443,7 +443,7 @@ class ServerCompletionQueue : public CompletionQueue { grpc_cq_polling_type polling_type_; friend class ::grpc::ServerBuilder; - friend class ::grpc_impl::Server; + friend class ::grpc::Server; }; } // namespace grpc diff --git a/include/grpcpp/impl/codegen/server_context_impl.h b/include/grpcpp/impl/codegen/server_context_impl.h index 75dbafa43a4..4312869925d 100644 --- a/include/grpcpp/impl/codegen/server_context_impl.h +++ b/include/grpcpp/impl/codegen/server_context_impl.h @@ -49,7 +49,6 @@ struct grpc_call; struct census_context; namespace grpc_impl { -class Server; template class ServerAsyncReader; template @@ -96,6 +95,7 @@ namespace grpc { class ClientContext; class CompletionQueue; class GenericServerContext; +class Server; class ServerInterface; #ifndef GRPC_CALLBACK_API_NONEXPERIMENTAL @@ -338,7 +338,7 @@ class ServerContextBase { friend class ::grpc::testing::ServerContextTestSpouse; friend class ::grpc::testing::DefaultReactorTestPeer; friend class ::grpc::ServerInterface; - friend class ::grpc_impl::Server; + friend class ::grpc::Server; template friend class ::grpc_impl::ServerAsyncReader; template @@ -544,7 +544,7 @@ class ServerContext : public ServerContextBase { private: // Constructor for internal use by server only - friend class ::grpc_impl::Server; + friend class ::grpc::Server; ServerContext(gpr_timespec deadline, grpc_metadata_array* arr) : ServerContextBase(deadline, arr) {} diff --git a/include/grpcpp/impl/codegen/service_type.h b/include/grpcpp/impl/codegen/service_type.h index 84a9893a7e8..05b3b1b8c6d 100644 --- a/include/grpcpp/impl/codegen/service_type.h +++ b/include/grpcpp/impl/codegen/service_type.h @@ -268,7 +268,7 @@ class Service { return methods_[idx]->handler(); } - friend class grpc_impl::Server; + friend class Server; friend class ServerInterface; ServerInterface* server_; std::vector> methods_; diff --git a/include/grpcpp/security/server_credentials.h b/include/grpcpp/security/server_credentials.h index 189f491a37d..a86d0e35f07 100644 --- a/include/grpcpp/security/server_credentials.h +++ b/include/grpcpp/security/server_credentials.h @@ -29,12 +29,9 @@ struct grpc_server; -namespace grpc_impl { - -class Server; -} // namespace grpc_impl namespace grpc { +class Server; /// Options to create ServerCredentials with SSL struct SslServerCredentialsOptions { /// \warning Deprecated @@ -72,7 +69,7 @@ class ServerCredentials { const std::shared_ptr& processor) = 0; private: - friend class ::grpc_impl::Server; + friend class Server; /// Tries to bind \a server to the given \a addr (eg, localhost:1234, /// 192.168.1.1:31416, [::1]:27182, etc.) diff --git a/include/grpcpp/server.h b/include/grpcpp/server.h index 3de2aba0b59..7437f85c383 100644 --- a/include/grpcpp/server.h +++ b/include/grpcpp/server.h @@ -1,6 +1,6 @@ /* * - * Copyright 2019 gRPC authors. + * Copyright 2015 gRPC authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,11 +19,359 @@ #ifndef GRPCPP_SERVER_H #define GRPCPP_SERVER_H -#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct grpc_server; + +namespace grpc_impl { +class ServerContext; +class ServerInitializer; +} // namespace grpc_impl namespace grpc { +class AsyncGenericService; + +namespace internal { +class ExternalConnectionAcceptorImpl; +} // namespace internal + +/// Represents a gRPC server. +/// +/// Use a \a grpc::ServerBuilder to create, configure, and start +/// \a Server instances. +class Server : public ServerInterface, private GrpcLibraryCodegen { + public: + ~Server(); + + /// Block until the server shuts down. + /// + /// \warning The server must be either shutting down or some other thread must + /// call \a Shutdown for this function to ever return. + void Wait() override; + + /// Global callbacks are a set of hooks that are called when server + /// events occur. \a SetGlobalCallbacks method is used to register + /// the hooks with gRPC. Note that + /// the \a GlobalCallbacks instance will be shared among all + /// \a Server instances in an application and can be set exactly + /// once per application. + class GlobalCallbacks { + public: + virtual ~GlobalCallbacks() {} + /// Called before server is created. + virtual void UpdateArguments(ChannelArguments* /*args*/) {} + /// Called before application callback for each synchronous server request + virtual void PreSynchronousRequest(grpc_impl::ServerContext* context) = 0; + /// Called after application callback for each synchronous server request + virtual void PostSynchronousRequest(grpc_impl::ServerContext* context) = 0; + /// Called before server is started. + virtual void PreServerStart(Server* /*server*/) {} + /// Called after a server port is added. + virtual void AddPort(Server* /*server*/, const std::string& /*addr*/, + ServerCredentials* /*creds*/, int /*port*/) {} + }; + /// Set the global callback object. Can only be called once per application. + /// Does not take ownership of callbacks, and expects the pointed to object + /// to be alive until all server objects in the process have been destroyed. + /// The same \a GlobalCallbacks object will be used throughout the + /// application and is shared among all \a Server objects. + static void SetGlobalCallbacks(GlobalCallbacks* callbacks); + + /// Returns a \em raw pointer to the underlying \a grpc_server instance. + /// EXPERIMENTAL: for internal/test use only + grpc_server* c_server(); + + /// Returns the health check service. + HealthCheckServiceInterface* GetHealthCheckService() const { + return health_check_service_.get(); + } + + /// Establish a channel for in-process communication + std::shared_ptr InProcessChannel(const ChannelArguments& args); + + /// NOTE: class experimental_type is not part of the public API of this class. + /// TODO(yashykt): Integrate into public API when this is no longer + /// experimental. + class experimental_type { + public: + explicit experimental_type(Server* server) : server_(server) {} + + /// Establish a channel for in-process communication with client + /// interceptors + std::shared_ptr InProcessChannelWithInterceptors( + const ChannelArguments& args, + std::vector< + std::unique_ptr> + interceptor_creators); + + private: + Server* server_; + }; + + /// NOTE: The function experimental() is not stable public API. It is a view + /// to the experimental components of this class. It may be changed or removed + /// at any time. + experimental_type experimental() { return experimental_type(this); } + + protected: + /// Register a service. This call does not take ownership of the service. + /// The service must exist for the lifetime of the Server instance. + bool RegisterService(const std::string* host, Service* service) override; + + /// Try binding the server to the given \a addr endpoint + /// (port, and optionally including IP address to bind to). + /// + /// It can be invoked multiple times. Should be used before + /// starting the server. + /// + /// \param addr The address to try to bind to the server (eg, localhost:1234, + /// 192.168.1.1:31416, [::1]:27182, etc.). + /// \param creds The credentials associated with the server. + /// + /// \return bound port number on success, 0 on failure. + /// + /// \warning It is an error to call this method on an already started server. + int AddListeningPort(const std::string& addr, + ServerCredentials* creds) override; + + /// NOTE: This is *NOT* a public API. The server constructors are supposed to + /// be used by \a ServerBuilder class only. The constructor will be made + /// 'private' very soon. + /// + /// Server constructors. To be used by \a ServerBuilder only. + /// + /// \param args The channel args + /// + /// \param sync_server_cqs The completion queues to use if the server is a + /// synchronous server (or a hybrid server). The server polls for new RPCs on + /// these queues + /// + /// \param min_pollers The minimum number of polling threads per server + /// completion queue (in param sync_server_cqs) to use for listening to + /// incoming requests (used only in case of sync server) + /// + /// \param max_pollers The maximum number of polling threads per server + /// completion queue (in param sync_server_cqs) to use for listening to + /// incoming requests (used only in case of sync server) + /// + /// \param sync_cq_timeout_msec The timeout to use when calling AsyncNext() on + /// server completion queues passed via sync_server_cqs param. + Server(ChannelArguments* args, + std::shared_ptr>> + sync_server_cqs, + int min_pollers, int max_pollers, int sync_cq_timeout_msec, + std::vector> + acceptors, + grpc_resource_quota* server_rq = nullptr, + std::vector< + std::unique_ptr> + interceptor_creators = std::vector>()); + + /// Start the server. + /// + /// \param cqs Completion queues for handling asynchronous services. The + /// caller is required to keep all completion queues live until the server is + /// destroyed. + /// \param num_cqs How many completion queues does \a cqs hold. + void Start(ServerCompletionQueue** cqs, size_t num_cqs) override; + + grpc_server* server() override { return server_; } + + protected: + /// NOTE: This method is not part of the public API for this class. + void set_health_check_service( + std::unique_ptr service) { + health_check_service_ = std::move(service); + } + + /// NOTE: This method is not part of the public API for this class. + bool health_check_service_disabled() const { + return health_check_service_disabled_; + } + + private: + std::vector>* + interceptor_creators() override { + return &interceptor_creators_; + } + + friend class AsyncGenericService; + friend class ServerBuilder; + friend class grpc_impl::ServerInitializer; + + class SyncRequest; + class CallbackRequestBase; + template + class CallbackRequest; + class UnimplementedAsyncRequest; + class UnimplementedAsyncResponse; + + /// SyncRequestThreadManager is an implementation of ThreadManager. This class + /// is responsible for polling for incoming RPCs and calling the RPC handlers. + /// This is only used in case of a Sync server (i.e a server exposing a sync + /// interface) + class SyncRequestThreadManager; + + /// Register a generic service. This call does not take ownership of the + /// service. The service must exist for the lifetime of the Server instance. + void RegisterAsyncGenericService(AsyncGenericService* service) override; + +#ifdef GRPC_CALLBACK_API_NONEXPERIMENTAL + /// Register a callback-based generic service. This call does not take + /// ownership of theservice. The service must exist for the lifetime of the + /// Server instance. + void RegisterCallbackGenericService(CallbackGenericService* service) override; +#else + /// NOTE: class experimental_registration_type is not part of the public API + /// of this class + /// TODO(vjpai): Move these contents to the public API of Server when + /// they are no longer experimental + class experimental_registration_type final + : public experimental_registration_interface { + public: + explicit experimental_registration_type(Server* server) : server_(server) {} + void RegisterCallbackGenericService( + experimental::CallbackGenericService* service) override { + server_->RegisterCallbackGenericService(service); + } + + private: + Server* server_; + }; + + /// TODO(vjpai): Mark this override when experimental type above is deleted + void RegisterCallbackGenericService( + experimental::CallbackGenericService* service); + + /// NOTE: The function experimental_registration() is not stable public API. + /// It is a view to the experimental components of this class. It may be + /// changed or removed at any time. + experimental_registration_interface* experimental_registration() override { + return &experimental_registration_; + } +#endif + + void PerformOpsOnCall(internal::CallOpSetInterface* ops, + internal::Call* call) override; + + void ShutdownInternal(gpr_timespec deadline) override; + + int max_receive_message_size() const override { + return max_receive_message_size_; + } + + CompletionQueue* CallbackCQ() override; + + grpc_impl::ServerInitializer* initializer(); + + // Functions to manage the server shutdown ref count. Things that increase + // the ref count are the running state of the server (take a ref at start and + // drop it at shutdown) and each running callback RPC. + void Ref(); + void UnrefWithPossibleNotify() /* LOCKS_EXCLUDED(mu_) */; + void UnrefAndWaitLocked() /* EXCLUSIVE_LOCKS_REQUIRED(mu_) */; + + std::vector> + acceptors_; + + // A vector of interceptor factory objects. + // This should be destroyed after health_check_service_ and this requirement + // is satisfied by declaring interceptor_creators_ before + // health_check_service_. (C++ mandates that member objects be destroyed in + // the reverse order of initialization.) + std::vector> + interceptor_creators_; + + int max_receive_message_size_; + + /// The following completion queues are ONLY used in case of Sync API + /// i.e. if the server has any services with sync methods. The server uses + /// these completion queues to poll for new RPCs + std::shared_ptr>> + sync_server_cqs_; + + /// List of \a ThreadManager instances (one for each cq in + /// the \a sync_server_cqs) + std::vector> sync_req_mgrs_; + +#ifndef GRPC_CALLBACK_API_NONEXPERIMENTAL + // For registering experimental callback generic service; remove when that + // method longer experimental + experimental_registration_type experimental_registration_{this}; +#endif + + // Server status + internal::Mutex mu_; + bool started_; + bool shutdown_; + bool shutdown_notified_; // Was notify called on the shutdown_cv_ + internal::CondVar shutdown_done_cv_; + bool shutdown_done_ = false; + std::atomic_int shutdown_refs_outstanding_{1}; + + internal::CondVar shutdown_cv_; + + std::shared_ptr global_callbacks_; + + std::vector services_; + bool has_async_generic_service_ = false; + bool has_callback_generic_service_ = false; + bool has_callback_methods_ = false; + + // Pointer to the wrapped grpc_server. + grpc_server* server_; + + std::unique_ptr server_initializer_; + + std::unique_ptr health_check_service_; + bool health_check_service_disabled_; + + // When appropriate, use a default callback generic service to handle + // unimplemented methods +#ifdef GRPC_CALLBACK_API_NONEXPERIMENTAL + std::unique_ptr unimplemented_service_; +#else + std::unique_ptr unimplemented_service_; +#endif + + // A special handler for resource exhausted in sync case + std::unique_ptr resource_exhausted_handler_; + + // Handler for callback generic service, if any + std::unique_ptr generic_handler_; + + // callback_cq_ references the callbackable completion queue associated + // with this server (if any). It is set on the first call to CallbackCQ(). + // It is _not owned_ by the server; ownership belongs with its internal + // shutdown callback tag (invoked when the CQ is fully shutdown). + CompletionQueue* callback_cq_ /* GUARDED_BY(mu_) */ = nullptr; -typedef ::grpc_impl::Server Server; + // List of CQs passed in by user that must be Shutdown only after Server is + // Shutdown. Even though this is only used with NDEBUG, instantiate it in all + // cases since otherwise the size will be inconsistent. + std::vector cq_list_; +}; } // namespace grpc diff --git a/include/grpcpp/server_impl.h b/include/grpcpp/server_impl.h deleted file mode 100644 index d121fc40993..00000000000 --- a/include/grpcpp/server_impl.h +++ /dev/null @@ -1,387 +0,0 @@ -/* - * - * Copyright 2015 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#ifndef GRPCPP_SERVER_IMPL_H -#define GRPCPP_SERVER_IMPL_H - -#include -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -struct grpc_server; - -namespace grpc { -class AsyncGenericService; - -namespace internal { -class ExternalConnectionAcceptorImpl; -} // namespace internal - -} // namespace grpc - -namespace grpc_impl { -class ServerContext; -class ServerInitializer; - -/// Represents a gRPC server. -/// -/// Use a \a grpc::ServerBuilder to create, configure, and start -/// \a Server instances. -class Server : public grpc::ServerInterface, private grpc::GrpcLibraryCodegen { - public: - ~Server(); - - /// Block until the server shuts down. - /// - /// \warning The server must be either shutting down or some other thread must - /// call \a Shutdown for this function to ever return. - void Wait() override; - - /// Global callbacks are a set of hooks that are called when server - /// events occur. \a SetGlobalCallbacks method is used to register - /// the hooks with gRPC. Note that - /// the \a GlobalCallbacks instance will be shared among all - /// \a Server instances in an application and can be set exactly - /// once per application. - class GlobalCallbacks { - public: - virtual ~GlobalCallbacks() {} - /// Called before server is created. - virtual void UpdateArguments(grpc::ChannelArguments* /*args*/) {} - /// Called before application callback for each synchronous server request - virtual void PreSynchronousRequest(grpc_impl::ServerContext* context) = 0; - /// Called after application callback for each synchronous server request - virtual void PostSynchronousRequest(grpc_impl::ServerContext* context) = 0; - /// Called before server is started. - virtual void PreServerStart(Server* /*server*/) {} - /// Called after a server port is added. - virtual void AddPort(Server* /*server*/, const std::string& /*addr*/, - grpc::ServerCredentials* /*creds*/, int /*port*/) {} - }; - /// Set the global callback object. Can only be called once per application. - /// Does not take ownership of callbacks, and expects the pointed to object - /// to be alive until all server objects in the process have been destroyed. - /// The same \a GlobalCallbacks object will be used throughout the - /// application and is shared among all \a Server objects. - static void SetGlobalCallbacks(GlobalCallbacks* callbacks); - - /// Returns a \em raw pointer to the underlying \a grpc_server instance. - /// EXPERIMENTAL: for internal/test use only - grpc_server* c_server(); - - /// Returns the health check service. - grpc::HealthCheckServiceInterface* GetHealthCheckService() const { - return health_check_service_.get(); - } - - /// Establish a channel for in-process communication - std::shared_ptr InProcessChannel( - const grpc::ChannelArguments& args); - - /// NOTE: class experimental_type is not part of the public API of this class. - /// TODO(yashykt): Integrate into public API when this is no longer - /// experimental. - class experimental_type { - public: - explicit experimental_type(Server* server) : server_(server) {} - - /// Establish a channel for in-process communication with client - /// interceptors - std::shared_ptr InProcessChannelWithInterceptors( - const grpc::ChannelArguments& args, - std::vector> - interceptor_creators); - - private: - Server* server_; - }; - - /// NOTE: The function experimental() is not stable public API. It is a view - /// to the experimental components of this class. It may be changed or removed - /// at any time. - experimental_type experimental() { return experimental_type(this); } - - protected: - /// Register a service. This call does not take ownership of the service. - /// The service must exist for the lifetime of the Server instance. - bool RegisterService(const std::string* host, - grpc::Service* service) override; - - /// Try binding the server to the given \a addr endpoint - /// (port, and optionally including IP address to bind to). - /// - /// It can be invoked multiple times. Should be used before - /// starting the server. - /// - /// \param addr The address to try to bind to the server (eg, localhost:1234, - /// 192.168.1.1:31416, [::1]:27182, etc.). - /// \param creds The credentials associated with the server. - /// - /// \return bound port number on success, 0 on failure. - /// - /// \warning It is an error to call this method on an already started server. - int AddListeningPort(const std::string& addr, - grpc::ServerCredentials* creds) override; - - /// NOTE: This is *NOT* a public API. The server constructors are supposed to - /// be used by \a ServerBuilder class only. The constructor will be made - /// 'private' very soon. - /// - /// Server constructors. To be used by \a ServerBuilder only. - /// - /// \param args The channel args - /// - /// \param sync_server_cqs The completion queues to use if the server is a - /// synchronous server (or a hybrid server). The server polls for new RPCs on - /// these queues - /// - /// \param min_pollers The minimum number of polling threads per server - /// completion queue (in param sync_server_cqs) to use for listening to - /// incoming requests (used only in case of sync server) - /// - /// \param max_pollers The maximum number of polling threads per server - /// completion queue (in param sync_server_cqs) to use for listening to - /// incoming requests (used only in case of sync server) - /// - /// \param sync_cq_timeout_msec The timeout to use when calling AsyncNext() on - /// server completion queues passed via sync_server_cqs param. - Server( - grpc::ChannelArguments* args, - std::shared_ptr>> - sync_server_cqs, - int min_pollers, int max_pollers, int sync_cq_timeout_msec, - std::vector< - std::shared_ptr> - acceptors, - grpc_resource_quota* server_rq = nullptr, - std::vector> - interceptor_creators = std::vector>()); - - /// Start the server. - /// - /// \param cqs Completion queues for handling asynchronous services. The - /// caller is required to keep all completion queues live until the server is - /// destroyed. - /// \param num_cqs How many completion queues does \a cqs hold. - void Start(grpc::ServerCompletionQueue** cqs, size_t num_cqs) override; - - grpc_server* server() override { return server_; } - - protected: - /// NOTE: This method is not part of the public API for this class. - void set_health_check_service( - std::unique_ptr service) { - health_check_service_ = std::move(service); - } - - /// NOTE: This method is not part of the public API for this class. - bool health_check_service_disabled() const { - return health_check_service_disabled_; - } - - private: - std::vector< - std::unique_ptr>* - interceptor_creators() override { - return &interceptor_creators_; - } - - friend class grpc::AsyncGenericService; - friend class grpc::ServerBuilder; - friend class grpc_impl::ServerInitializer; - - class SyncRequest; - class CallbackRequestBase; - template - class CallbackRequest; - class UnimplementedAsyncRequest; - class UnimplementedAsyncResponse; - - /// SyncRequestThreadManager is an implementation of ThreadManager. This class - /// is responsible for polling for incoming RPCs and calling the RPC handlers. - /// This is only used in case of a Sync server (i.e a server exposing a sync - /// interface) - class SyncRequestThreadManager; - - /// Register a generic service. This call does not take ownership of the - /// service. The service must exist for the lifetime of the Server instance. - void RegisterAsyncGenericService(grpc::AsyncGenericService* service) override; - -#ifdef GRPC_CALLBACK_API_NONEXPERIMENTAL - /// Register a callback-based generic service. This call does not take - /// ownership of theservice. The service must exist for the lifetime of the - /// Server instance. - void RegisterCallbackGenericService( - grpc::CallbackGenericService* service) override; -#else - /// NOTE: class experimental_registration_type is not part of the public API - /// of this class - /// TODO(vjpai): Move these contents to the public API of Server when - /// they are no longer experimental - class experimental_registration_type final - : public experimental_registration_interface { - public: - explicit experimental_registration_type(Server* server) : server_(server) {} - void RegisterCallbackGenericService( - grpc::experimental::CallbackGenericService* service) override { - server_->RegisterCallbackGenericService(service); - } - - private: - Server* server_; - }; - - /// TODO(vjpai): Mark this override when experimental type above is deleted - void RegisterCallbackGenericService( - grpc::experimental::CallbackGenericService* service); - - /// NOTE: The function experimental_registration() is not stable public API. - /// It is a view to the experimental components of this class. It may be - /// changed or removed at any time. - experimental_registration_interface* experimental_registration() override { - return &experimental_registration_; - } -#endif - - void PerformOpsOnCall(grpc::internal::CallOpSetInterface* ops, - grpc::internal::Call* call) override; - - void ShutdownInternal(gpr_timespec deadline) override; - - int max_receive_message_size() const override { - return max_receive_message_size_; - } - - grpc::CompletionQueue* CallbackCQ() override; - - grpc_impl::ServerInitializer* initializer(); - - // Functions to manage the server shutdown ref count. Things that increase - // the ref count are the running state of the server (take a ref at start and - // drop it at shutdown) and each running callback RPC. - void Ref(); - void UnrefWithPossibleNotify() /* LOCKS_EXCLUDED(mu_) */; - void UnrefAndWaitLocked() /* EXCLUSIVE_LOCKS_REQUIRED(mu_) */; - - std::vector> - acceptors_; - - // A vector of interceptor factory objects. - // This should be destroyed after health_check_service_ and this requirement - // is satisfied by declaring interceptor_creators_ before - // health_check_service_. (C++ mandates that member objects be destroyed in - // the reverse order of initialization.) - std::vector< - std::unique_ptr> - interceptor_creators_; - - int max_receive_message_size_; - - /// The following completion queues are ONLY used in case of Sync API - /// i.e. if the server has any services with sync methods. The server uses - /// these completion queues to poll for new RPCs - std::shared_ptr>> - sync_server_cqs_; - - /// List of \a ThreadManager instances (one for each cq in - /// the \a sync_server_cqs) - std::vector> sync_req_mgrs_; - -#ifndef GRPC_CALLBACK_API_NONEXPERIMENTAL - // For registering experimental callback generic service; remove when that - // method longer experimental - experimental_registration_type experimental_registration_{this}; -#endif - - // Server status - grpc::internal::Mutex mu_; - bool started_; - bool shutdown_; - bool shutdown_notified_; // Was notify called on the shutdown_cv_ - grpc::internal::CondVar shutdown_done_cv_; - bool shutdown_done_ = false; - std::atomic_int shutdown_refs_outstanding_{1}; - - grpc::internal::CondVar shutdown_cv_; - - std::shared_ptr global_callbacks_; - - std::vector services_; - bool has_async_generic_service_ = false; - bool has_callback_generic_service_ = false; - bool has_callback_methods_ = false; - - // Pointer to the wrapped grpc_server. - grpc_server* server_; - - std::unique_ptr server_initializer_; - - std::unique_ptr health_check_service_; - bool health_check_service_disabled_; - - // When appropriate, use a default callback generic service to handle - // unimplemented methods -#ifdef GRPC_CALLBACK_API_NONEXPERIMENTAL - std::unique_ptr unimplemented_service_; -#else - std::unique_ptr - unimplemented_service_; -#endif - - // A special handler for resource exhausted in sync case - std::unique_ptr resource_exhausted_handler_; - - // Handler for callback generic service, if any - std::unique_ptr generic_handler_; - - // callback_cq_ references the callbackable completion queue associated - // with this server (if any). It is set on the first call to CallbackCQ(). - // It is _not owned_ by the server; ownership belongs with its internal - // shutdown callback tag (invoked when the CQ is fully shutdown). - grpc::CompletionQueue* callback_cq_ /* GUARDED_BY(mu_) */ = nullptr; - - // List of CQs passed in by user that must be Shutdown only after Server is - // Shutdown. Even though this is only used with NDEBUG, instantiate it in all - // cases since otherwise the size will be inconsistent. - std::vector cq_list_; -}; - -} // namespace grpc_impl - -#endif // GRPCPP_SERVER_IMPL_H diff --git a/src/cpp/server/server_cc.cc b/src/cpp/server/server_cc.cc index 1228b0f287e..a2adee507c1 100644 --- a/src/cpp/server/server_cc.cc +++ b/src/cpp/server/server_cc.cc @@ -283,10 +283,6 @@ class ShutdownCallback : public grpc_experimental_completion_queue_functor { }; } // namespace -} // namespace grpc - -namespace grpc_impl { - /// Use private inheritance rather than composition only to establish order /// of construction, since the public base class should be constructed after the /// elements belonging to the private base class are constructed. This is not @@ -1361,4 +1357,4 @@ grpc::CompletionQueue* Server::CallbackCQ() { return callback_cq_; } -} // namespace grpc_impl +} // namespace grpc diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index af4ca9e6547..dacc1c16714 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -1024,7 +1024,6 @@ include/grpcpp/security/tls_credentials_options.h \ include/grpcpp/server.h \ include/grpcpp/server_builder.h \ include/grpcpp/server_context.h \ -include/grpcpp/server_impl.h \ include/grpcpp/server_posix.h \ include/grpcpp/support/async_stream.h \ include/grpcpp/support/async_stream_impl.h \ diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index b645161be23..7b38d95dd01 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -1024,7 +1024,6 @@ include/grpcpp/security/tls_credentials_options.h \ include/grpcpp/server.h \ include/grpcpp/server_builder.h \ include/grpcpp/server_context.h \ -include/grpcpp/server_impl.h \ include/grpcpp/server_posix.h \ include/grpcpp/support/async_stream.h \ include/grpcpp/support/async_stream_impl.h \ From f700a8c73c71f4117ac6367fbf1929eb25473e80 Mon Sep 17 00:00:00 2001 From: Menghan Li Date: Thu, 20 Aug 2020 16:19:55 -0700 Subject: [PATCH 11/62] Merge pull request #23909 from menghanl/xds_test_fail_on_error --- tools/run_tests/run_xds_tests.py | 33 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/tools/run_tests/run_xds_tests.py b/tools/run_tests/run_xds_tests.py index 6f02a0daa1e..5cdac1f23dc 100755 --- a/tools/run_tests/run_xds_tests.py +++ b/tools/run_tests/run_xds_tests.py @@ -1750,25 +1750,20 @@ try: # metadata arg is not specified. metadata_to_send = '' - if test_case in _TESTS_TO_FAIL_ON_RPC_FAILURE: - # TODO(ericgribkoff) Unconditional wait is recommended by TD - # team when reusing backend resources after config changes - # between test cases, as we are doing here. This should address - # flakiness issues with these tests; other attempts to deflake - # (such as waiting for the first successful RPC before failing - # on any subsequent failures) were insufficient because, due to - # propagation delays, we may initially see an RPC succeed to the - # expected backends but due to a stale configuration: e.g., test - # A (1) routes traffic to MIG A, then (2) switches to MIG B, - # then (3) back to MIG A. Test B begins running and sees RPCs - # going to MIG A, as expected. However, due to propagation - # delays, Test B is actually seeing the stale config from step - # (1), and then fails when it gets update (2) unexpectedly - # switching to MIG B. - time.sleep(200) - fail_on_failed_rpc = '--fail_on_failed_rpc=true' - else: - fail_on_failed_rpc = '--fail_on_failed_rpc=false' + # TODO(ericgribkoff) Temporarily disable fail_on_failed_rpc checks + # in the client. This means we will ignore intermittent RPC + # failures (but this framework still checks that the final result + # is as expected). + # + # Reason for disabling this is, the resources are shared by + # multiple tests, and a change in previous test could be delayed + # until the second test starts. The second test may see + # intermittent failures because of that. + # + # A fix is to not share resources between tests (though that does + # mean the tests will be significantly slower due to creating new + # resources). + fail_on_failed_rpc = '' client_cmd_formatted = args.client_cmd.format( server_uri=server_uri, From 8fd8a47c7dce836f57ae76cfa1e4ffb6e93be22a Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Thu, 20 Aug 2020 16:43:14 -0700 Subject: [PATCH 12/62] Merge pull request #23911 from ericgribkoff/remove_flake_fix Account for empty EDS update in test_round_robin --- tools/run_tests/run_xds_tests.py | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/tools/run_tests/run_xds_tests.py b/tools/run_tests/run_xds_tests.py index 5cdac1f23dc..f011a83f577 100755 --- a/tools/run_tests/run_xds_tests.py +++ b/tools/run_tests/run_xds_tests.py @@ -566,17 +566,27 @@ def test_round_robin(gcp, backend_service, instance_group): threshold = 1 wait_until_all_rpcs_go_to_given_backends(instance_names, _WAIT_FOR_STATS_SEC) - stats = get_client_stats(_NUM_TEST_RPCS, _WAIT_FOR_STATS_SEC) - requests_received = [stats.rpcs_by_peer[x] for x in stats.rpcs_by_peer] - total_requests_received = sum(requests_received) - if total_requests_received != _NUM_TEST_RPCS: - raise Exception('Unexpected RPC failures', stats) - expected_requests = total_requests_received / len(instance_names) - for instance in instance_names: - if abs(stats.rpcs_by_peer[instance] - expected_requests) > threshold: - raise Exception( - 'RPC peer distribution differs from expected by more than %d ' - 'for instance %s (%s)', threshold, instance, stats) + # TODO(ericgribkoff) Delayed config propagation from earlier tests + # may result in briefly receiving an empty EDS update, resulting in failed + # RPCs. Retry distribution validation if this occurs; long-term fix is + # creating new backend resources for each individual test case. + max_attempts = 10 + for i in range(max_attempts): + stats = get_client_stats(_NUM_TEST_RPCS, _WAIT_FOR_STATS_SEC) + requests_received = [stats.rpcs_by_peer[x] for x in stats.rpcs_by_peer] + total_requests_received = sum(requests_received) + if total_requests_received != _NUM_TEST_RPCS: + logger.info('Unexpected RPC failures, retrying: %s', stats) + continue + expected_requests = total_requests_received / len(instance_names) + for instance in instance_names: + if abs(stats.rpcs_by_peer[instance] - + expected_requests) > threshold: + raise Exception( + 'RPC peer distribution differs from expected by more than %d ' + 'for instance %s (%s)' % (threshold, instance, stats)) + return + raise Exception('RPC failures persisted through %d retries' % max_attempts) def test_secondary_locality_gets_no_requests_on_partial_primary_failure( From efb7b0f7f83df29e517737d316506a9d7ecccbd5 Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Thu, 20 Aug 2020 16:45:26 -0700 Subject: [PATCH 13/62] Merge pull request #23915 from ericgribkoff/really_remove_flake_fix Account for connecting to different zone TD --- tools/run_tests/run_xds_tests.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/tools/run_tests/run_xds_tests.py b/tools/run_tests/run_xds_tests.py index f011a83f577..3e4c39b5d6a 100755 --- a/tools/run_tests/run_xds_tests.py +++ b/tools/run_tests/run_xds_tests.py @@ -546,12 +546,30 @@ def test_remove_instance_group(gcp, backend_service, instance_group, instance_names = get_instance_names(gcp, instance_group) same_zone_instance_names = get_instance_names(gcp, same_zone_instance_group) - wait_until_all_rpcs_go_to_given_backends( - instance_names + same_zone_instance_names, _WAIT_FOR_BACKEND_SEC) + try: + wait_until_all_rpcs_go_to_given_backends( + instance_names + same_zone_instance_names, + _WAIT_FOR_OPERATION_SEC) + remaining_instance_group = same_zone_instance_group + remaining_instance_names = same_zone_instance_names + except RpcDistributionError as e: + # If connected to TD in a different zone, we may route traffic to + # only one instance group. Determine which group that is to continue + # with the remainder of the test case. + try: + wait_until_all_rpcs_go_to_given_backends( + instance_names, _WAIT_FOR_STATS_SEC) + remaining_instance_group = same_zone_instance_group + remaining_instance_names = same_zone_instance_names + except RpcDistributionError as e: + wait_until_all_rpcs_go_to_given_backends( + same_zone_instance_names, _WAIT_FOR_STATS_SEC) + remaining_instance_group = instance_group + remaining_instance_names = instance_names patch_backend_instances(gcp, - backend_service, [same_zone_instance_group], + backend_service, [remaining_instance_group], balancing_mode='RATE') - wait_until_all_rpcs_go_to_given_backends(same_zone_instance_names, + wait_until_all_rpcs_go_to_given_backends(remaining_instance_names, _WAIT_FOR_BACKEND_SEC) finally: patch_backend_instances(gcp, backend_service, [instance_group]) From 26de5087d80ae595f218a5c148c815dda4d50fb8 Mon Sep 17 00:00:00 2001 From: Menghan Li Date: Thu, 20 Aug 2020 21:24:52 -0700 Subject: [PATCH 14/62] Merge pull request #23924 from menghanl/disable_new_instance_group_test xds testing: delete test new_instance_group --- tools/run_tests/run_xds_tests.py | 35 +------------------------------- 1 file changed, 1 insertion(+), 34 deletions(-) diff --git a/tools/run_tests/run_xds_tests.py b/tools/run_tests/run_xds_tests.py index 3e4c39b5d6a..0746faa66d4 100755 --- a/tools/run_tests/run_xds_tests.py +++ b/tools/run_tests/run_xds_tests.py @@ -48,7 +48,6 @@ _TEST_CASES = [ 'backends_restart', 'change_backend_service', 'gentle_failover', - 'new_instance_group_receives_traffic', 'ping_pong', 'remove_instance_group', 'round_robin', @@ -247,9 +246,7 @@ _BOOTSTRAP_TEMPLATE = """ # TODO(ericgribkoff) Add change_backend_service to this list once TD no longer # sends an update with no localities when adding the MIG to the backend service # can race with the URL map patch. -_TESTS_TO_FAIL_ON_RPC_FAILURE = [ - 'new_instance_group_receives_traffic', 'ping_pong', 'round_robin' -] +_TESTS_TO_FAIL_ON_RPC_FAILURE = ['ping_pong', 'round_robin'] # Tests that run UnaryCall and EmptyCall. _TESTS_TO_RUN_MULTIPLE_RPCS = ['path_matching', 'header_matching'] # Tests that make UnaryCall with test metadata. @@ -498,32 +495,6 @@ def test_gentle_failover(gcp, _WAIT_FOR_BACKEND_SEC) -def test_new_instance_group_receives_traffic(gcp, backend_service, - instance_group, - same_zone_instance_group): - logger.info('Running test_new_instance_group_receives_traffic') - instance_names = get_instance_names(gcp, instance_group) - # TODO(ericgribkoff) Reduce this timeout. When running sequentially, this - # occurs after patching the url map in test_change_backend_service, so we - # need the extended timeout here as well. - wait_until_all_rpcs_go_to_given_backends(instance_names, - _WAIT_FOR_URL_MAP_PATCH_SEC) - try: - patch_backend_instances(gcp, - backend_service, - [instance_group, same_zone_instance_group], - balancing_mode='RATE') - wait_for_healthy_backends(gcp, backend_service, instance_group) - wait_for_healthy_backends(gcp, backend_service, - same_zone_instance_group) - combined_instance_names = instance_names + get_instance_names( - gcp, same_zone_instance_group) - wait_until_all_rpcs_go_to_given_backends(combined_instance_names, - _WAIT_FOR_BACKEND_SEC) - finally: - patch_backend_instances(gcp, backend_service, [instance_group]) - - def test_ping_pong(gcp, backend_service, instance_group): logger.info('Running test_ping_pong') wait_for_healthy_backends(gcp, backend_service, instance_group) @@ -1817,10 +1788,6 @@ try: elif test_case == 'gentle_failover': test_gentle_failover(gcp, backend_service, instance_group, secondary_zone_instance_group) - elif test_case == 'new_instance_group_receives_traffic': - test_new_instance_group_receives_traffic( - gcp, backend_service, instance_group, - same_zone_instance_group) elif test_case == 'ping_pong': test_ping_pong(gcp, backend_service, instance_group) elif test_case == 'remove_instance_group': From 2c0d5f70e023aad7c6a7983e428f17f01601928f Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Thu, 20 Aug 2020 21:49:02 -0700 Subject: [PATCH 15/62] Merge pull request #23921 from ericgribkoff/bump_timeout Increase xDS job timeout while increased run time is investigated --- tools/internal_ci/linux/grpc_xds.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/internal_ci/linux/grpc_xds.cfg b/tools/internal_ci/linux/grpc_xds.cfg index 107404a30e9..ed3e0f563b8 100644 --- a/tools/internal_ci/linux/grpc_xds.cfg +++ b/tools/internal_ci/linux/grpc_xds.cfg @@ -16,7 +16,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/linux/grpc_xds.sh" -timeout_mins: 90 +timeout_mins: 120 env_vars { key: "BAZEL_SCRIPT" value: "tools/internal_ci/linux/grpc_xds_bazel_test_in_docker.sh" From ccc8f505049692dbc296ec814cb12cda4a69a061 Mon Sep 17 00:00:00 2001 From: Menghan Li Date: Fri, 21 Aug 2020 09:45:16 -0700 Subject: [PATCH 16/62] Merge pull request #23923 from menghanl/xds_test_set_serving_status --- .../grpc_xds_bazel_python_test_in_docker.sh | 12 +++++++++ .../linux/grpc_xds_bazel_test_in_docker.sh | 12 +++++++++ .../linux/grpc_xds_csharp_test_in_docker.sh | 12 +++++++++ .../linux/grpc_xds_php_test_in_docker.sh | 12 +++++++++ .../linux/grpc_xds_ruby_test_in_docker.sh | 12 +++++++++ tools/run_tests/helper_scripts/prep_xds.sh | 10 +++++++ tools/run_tests/run_xds_tests.py | 26 ++++++++++++++++--- 7 files changed, 92 insertions(+), 4 deletions(-) diff --git a/tools/internal_ci/linux/grpc_xds_bazel_python_test_in_docker.sh b/tools/internal_ci/linux/grpc_xds_bazel_python_test_in_docker.sh index 0fe64ebac52..b2426e92ad6 100755 --- a/tools/internal_ci/linux/grpc_xds_bazel_python_test_in_docker.sh +++ b/tools/internal_ci/linux/grpc_xds_bazel_python_test_in_docker.sh @@ -46,6 +46,18 @@ touch "$TOOLS_DIR"/src/proto/grpc/testing/__init__.py "$PROTO_SOURCE_DIR"/messages.proto \ "$PROTO_SOURCE_DIR"/empty.proto +HEALTH_PROTO_SOURCE_DIR=src/proto/grpc/health/v1 +HEALTH_PROTO_DEST_DIR=${TOOLS_DIR}/${HEALTH_PROTO_SOURCE_DIR} +mkdir -p ${HEALTH_PROTO_DEST_DIR} +touch "$TOOLS_DIR"/src/proto/grpc/health/__init__.py +touch "$TOOLS_DIR"/src/proto/grpc/health/v1/__init__.py + +"$PYTHON" -m grpc_tools.protoc \ + --proto_path=. \ + --python_out=${TOOLS_DIR} \ + --grpc_python_out=${TOOLS_DIR} \ + ${HEALTH_PROTO_SOURCE_DIR}/health.proto + bazel build //src/python/grpcio_tests/tests_py3_only/interop:xds_interop_client GRPC_VERBOSITY=debug GRPC_TRACE=xds_client,xds_resolver,xds_routing_lb,cds_lb,eds_lb,priority_lb,weighted_target_lb,lrs_lb "$PYTHON" \ diff --git a/tools/internal_ci/linux/grpc_xds_bazel_test_in_docker.sh b/tools/internal_ci/linux/grpc_xds_bazel_test_in_docker.sh index 9ed6eb7e0ad..69062cc8f49 100755 --- a/tools/internal_ci/linux/grpc_xds_bazel_test_in_docker.sh +++ b/tools/internal_ci/linux/grpc_xds_bazel_test_in_docker.sh @@ -46,6 +46,18 @@ touch "$TOOLS_DIR"/src/proto/grpc/testing/__init__.py "$PROTO_SOURCE_DIR"/messages.proto \ "$PROTO_SOURCE_DIR"/empty.proto +HEALTH_PROTO_SOURCE_DIR=src/proto/grpc/health/v1 +HEALTH_PROTO_DEST_DIR=${TOOLS_DIR}/${HEALTH_PROTO_SOURCE_DIR} +mkdir -p ${HEALTH_PROTO_DEST_DIR} +touch "$TOOLS_DIR"/src/proto/grpc/health/__init__.py +touch "$TOOLS_DIR"/src/proto/grpc/health/v1/__init__.py + +"$PYTHON" -m grpc_tools.protoc \ + --proto_path=. \ + --python_out=${TOOLS_DIR} \ + --grpc_python_out=${TOOLS_DIR} \ + ${HEALTH_PROTO_SOURCE_DIR}/health.proto + bazel build test/cpp/interop:xds_interop_client # Test cases "path_matching" and "header_matching" are not included in "all", diff --git a/tools/internal_ci/linux/grpc_xds_csharp_test_in_docker.sh b/tools/internal_ci/linux/grpc_xds_csharp_test_in_docker.sh index f9c9a8ad0f5..06f695b2b82 100755 --- a/tools/internal_ci/linux/grpc_xds_csharp_test_in_docker.sh +++ b/tools/internal_ci/linux/grpc_xds_csharp_test_in_docker.sh @@ -46,6 +46,18 @@ touch "$TOOLS_DIR"/src/proto/grpc/testing/__init__.py "$PROTO_SOURCE_DIR"/messages.proto \ "$PROTO_SOURCE_DIR"/empty.proto +HEALTH_PROTO_SOURCE_DIR=src/proto/grpc/health/v1 +HEALTH_PROTO_DEST_DIR=${TOOLS_DIR}/${HEALTH_PROTO_SOURCE_DIR} +mkdir -p ${HEALTH_PROTO_DEST_DIR} +touch "$TOOLS_DIR"/src/proto/grpc/health/__init__.py +touch "$TOOLS_DIR"/src/proto/grpc/health/v1/__init__.py + +"$PYTHON" -m grpc_tools.protoc \ + --proto_path=. \ + --python_out=${TOOLS_DIR} \ + --grpc_python_out=${TOOLS_DIR} \ + ${HEALTH_PROTO_SOURCE_DIR}/health.proto + python tools/run_tests/run_tests.py -l csharp -c opt --build_only # Test cases "path_matching" and "header_matching" are not included in "all", diff --git a/tools/internal_ci/linux/grpc_xds_php_test_in_docker.sh b/tools/internal_ci/linux/grpc_xds_php_test_in_docker.sh index 2ff670fd7a1..c9c347901f8 100755 --- a/tools/internal_ci/linux/grpc_xds_php_test_in_docker.sh +++ b/tools/internal_ci/linux/grpc_xds_php_test_in_docker.sh @@ -46,6 +46,18 @@ touch "$TOOLS_DIR"/src/proto/grpc/testing/__init__.py "$PROTO_SOURCE_DIR"/messages.proto \ "$PROTO_SOURCE_DIR"/empty.proto +HEALTH_PROTO_SOURCE_DIR=src/proto/grpc/health/v1 +HEALTH_PROTO_DEST_DIR=${TOOLS_DIR}/${HEALTH_PROTO_SOURCE_DIR} +mkdir -p ${HEALTH_PROTO_DEST_DIR} +touch "$TOOLS_DIR"/src/proto/grpc/health/__init__.py +touch "$TOOLS_DIR"/src/proto/grpc/health/v1/__init__.py + +"$PYTHON" -m grpc_tools.protoc \ + --proto_path=. \ + --python_out=${TOOLS_DIR} \ + --grpc_python_out=${TOOLS_DIR} \ + ${HEALTH_PROTO_SOURCE_DIR}/health.proto + # Generate and compile the PHP extension. (pear package && \ find . -name grpc-*.tgz | xargs -I{} pecl install {}) diff --git a/tools/internal_ci/linux/grpc_xds_ruby_test_in_docker.sh b/tools/internal_ci/linux/grpc_xds_ruby_test_in_docker.sh index de2ab4b58cd..0776a0c4c0e 100644 --- a/tools/internal_ci/linux/grpc_xds_ruby_test_in_docker.sh +++ b/tools/internal_ci/linux/grpc_xds_ruby_test_in_docker.sh @@ -46,6 +46,18 @@ touch "$TOOLS_DIR"/src/proto/grpc/testing/__init__.py "$PROTO_SOURCE_DIR"/messages.proto \ "$PROTO_SOURCE_DIR"/empty.proto +HEALTH_PROTO_SOURCE_DIR=src/proto/grpc/health/v1 +HEALTH_PROTO_DEST_DIR=${TOOLS_DIR}/${HEALTH_PROTO_SOURCE_DIR} +mkdir -p ${HEALTH_PROTO_DEST_DIR} +touch "$TOOLS_DIR"/src/proto/grpc/health/__init__.py +touch "$TOOLS_DIR"/src/proto/grpc/health/v1/__init__.py + +"$PYTHON" -m grpc_tools.protoc \ + --proto_path=. \ + --python_out=${TOOLS_DIR} \ + --grpc_python_out=${TOOLS_DIR} \ + ${HEALTH_PROTO_SOURCE_DIR}/health.proto + (cd src/ruby && bundle && rake compile) GRPC_VERBOSITY=debug GRPC_TRACE=xds_client,xds_resolver,xds_routing_lb,cds_lb,eds_lb,priority_lb,weighted_target_lb,lrs_lb "$PYTHON" \ diff --git a/tools/run_tests/helper_scripts/prep_xds.sh b/tools/run_tests/helper_scripts/prep_xds.sh index ab15e2eb1c8..9d668bb68c6 100755 --- a/tools/run_tests/helper_scripts/prep_xds.sh +++ b/tools/run_tests/helper_scripts/prep_xds.sh @@ -35,3 +35,13 @@ python3 -m grpc_tools.protoc \ ${PROTO_SOURCE_DIR}/test.proto \ ${PROTO_SOURCE_DIR}/messages.proto \ ${PROTO_SOURCE_DIR}/empty.proto + +HEALTH_PROTO_SOURCE_DIR=src/proto/grpc/health/v1 +HEALTH_PROTO_DEST_DIR=${TOOLS_DIR}/${HEALTH_PROTO_SOURCE_DIR} +mkdir -p ${HEALTH_PROTO_DEST_DIR} + +python3 -m grpc_tools.protoc \ + --proto_path=. \ + --python_out=${TOOLS_DIR} \ + --grpc_python_out=${TOOLS_DIR} \ + ${HEALTH_PROTO_SOURCE_DIR}/health.proto diff --git a/tools/run_tests/run_xds_tests.py b/tools/run_tests/run_xds_tests.py index 0746faa66d4..f7702edb566 100755 --- a/tools/run_tests/run_xds_tests.py +++ b/tools/run_tests/run_xds_tests.py @@ -32,6 +32,8 @@ from oauth2client.client import GoogleCredentials import python_utils.jobset as jobset import python_utils.report_utils as report_utils +from src.proto.grpc.health.v1 import health_pb2 +from src.proto.grpc.health.v1 import health_pb2_grpc from src.proto.grpc.testing import empty_pb2 from src.proto.grpc.testing import messages_pb2 from src.proto.grpc.testing import test_pb2_grpc @@ -964,14 +966,30 @@ def test_header_matching(gcp, original_backend_service, instance_group, def set_serving_status(instances, service_port, serving): + logger.info('setting %s serving status to %s', instances, serving) for instance in instances: with grpc.insecure_channel('%s:%d' % (instance, service_port)) as channel: + logger.info('setting %s serving status to %s', instance, serving) stub = test_pb2_grpc.XdsUpdateHealthServiceStub(channel) - if serving: - stub.SetServing(empty_pb2.Empty()) - else: - stub.SetNotServing(empty_pb2.Empty()) + health_stub = health_pb2_grpc.HealthStub(channel) + + retry_count = 5 + for i in range(5): + if serving: + stub.SetServing(empty_pb2.Empty()) + else: + stub.SetNotServing(empty_pb2.Empty()) + serving_status = health_stub.Check( + health_pb2.HealthCheckRequest()) + logger.info('got instance service status %s', serving_status) + want_status = health_pb2.HealthCheckResponse.SERVING if serving else health_pb2.HealthCheckResponse.NOT_SERVING + if serving_status.status == want_status: + break + if i == retry_count - 1: + raise Exception( + 'failed to set instance service status after %d retries' + % retry_count) def is_primary_instance_group(gcp, instance_group): From 6b29a6d7b27f003eb5217b3b8abc3c8d9bbd76c2 Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Fri, 21 Aug 2020 10:07:04 -0700 Subject: [PATCH 17/62] Merge pull request #23928 from ericgribkoff/give_round_robin_more_time Increase attempt number waiting for round_robin config propagation --- tools/run_tests/run_xds_tests.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/run_tests/run_xds_tests.py b/tools/run_tests/run_xds_tests.py index f7702edb566..8822c9e1b0d 100755 --- a/tools/run_tests/run_xds_tests.py +++ b/tools/run_tests/run_xds_tests.py @@ -561,7 +561,9 @@ def test_round_robin(gcp, backend_service, instance_group): # may result in briefly receiving an empty EDS update, resulting in failed # RPCs. Retry distribution validation if this occurs; long-term fix is # creating new backend resources for each individual test case. - max_attempts = 10 + # Each attempt takes 10 seconds. Config propagation can take several + # minutes. + max_attempts = 40 for i in range(max_attempts): stats = get_client_stats(_NUM_TEST_RPCS, _WAIT_FOR_STATS_SEC) requests_received = [stats.rpcs_by_peer[x] for x in stats.rpcs_by_peer] From 005b57b0a9bb0b5259dc57a4fc771fe20c085c1f Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Fri, 21 Aug 2020 10:10:08 -0700 Subject: [PATCH 18/62] Merge pull request #23927 from ericgribkoff/disable_dist_check xds testing: remove distribution comparison in backends restart --- tools/run_tests/run_xds_tests.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tools/run_tests/run_xds_tests.py b/tools/run_tests/run_xds_tests.py index 8822c9e1b0d..da5a9f85dee 100755 --- a/tools/run_tests/run_xds_tests.py +++ b/tools/run_tests/run_xds_tests.py @@ -396,7 +396,6 @@ def test_backends_restart(gcp, backend_service, instance_group): start_time = time.time() wait_until_all_rpcs_go_to_given_backends(instance_names, _WAIT_FOR_STATS_SEC) - stats = get_client_stats(_NUM_TEST_RPCS, _WAIT_FOR_STATS_SEC) try: resize_instance_group(gcp, instance_group, 0) wait_until_all_rpcs_go_to_given_backends_or_fail([], @@ -407,15 +406,6 @@ def test_backends_restart(gcp, backend_service, instance_group): new_instance_names = get_instance_names(gcp, instance_group) wait_until_all_rpcs_go_to_given_backends(new_instance_names, _WAIT_FOR_BACKEND_SEC) - new_stats = get_client_stats(_NUM_TEST_RPCS, _WAIT_FOR_STATS_SEC) - original_distribution = list(stats.rpcs_by_peer.values()) - original_distribution.sort() - new_distribution = list(new_stats.rpcs_by_peer.values()) - new_distribution.sort() - threshold = 3 - for i in range(len(original_distribution)): - if abs(original_distribution[i] - new_distribution[i]) > threshold: - raise Exception('Distributions do not match: ', stats, new_stats) def test_change_backend_service(gcp, original_backend_service, instance_group, From 5183c76feffb35465c1e0f7105d90d893d1352d2 Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Fri, 21 Aug 2020 13:46:50 -0700 Subject: [PATCH 19/62] Merge pull request #23939 from ericgribkoff/bump_ops_timeout Increase timeout for GCP operations --- tools/run_tests/run_xds_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/run_xds_tests.py b/tools/run_tests/run_xds_tests.py index da5a9f85dee..14f7cab1364 100755 --- a/tools/run_tests/run_xds_tests.py +++ b/tools/run_tests/run_xds_tests.py @@ -215,7 +215,7 @@ if args.verbose: _DEFAULT_SERVICE_PORT = 80 _WAIT_FOR_BACKEND_SEC = args.wait_for_backend_sec -_WAIT_FOR_OPERATION_SEC = 300 +_WAIT_FOR_OPERATION_SEC = 1200 _INSTANCE_GROUP_SIZE = args.instance_group_size _NUM_TEST_RPCS = 10 * args.qps _WAIT_FOR_STATS_SEC = 180 From 7e684381ba82aaab961938fc1ce6070a62aff891 Mon Sep 17 00:00:00 2001 From: Hannah Shi Date: Fri, 17 Jul 2020 22:34:35 +0000 Subject: [PATCH 20/62] avoid destroy channel more than once --- src/php/ext/grpc/channel.c | 2 +- src/php/ext/grpc/php_grpc.c | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/php/ext/grpc/channel.c b/src/php/ext/grpc/channel.c index 47483812b25..e9b6d67f817 100644 --- a/src/php/ext/grpc/channel.c +++ b/src/php/ext/grpc/channel.c @@ -50,7 +50,7 @@ extern HashTable grpc_persistent_list; extern HashTable grpc_target_upper_bound_map; void free_grpc_channel_wrapper(grpc_channel_wrapper* channel, bool free_channel) { - if (free_channel) { + if (free_channel && channel->wrapped) { grpc_channel_destroy(channel->wrapped); channel->wrapped = NULL; } diff --git a/src/php/ext/grpc/php_grpc.c b/src/php/ext/grpc/php_grpc.c index f567d8c59c3..01764a71202 100644 --- a/src/php/ext/grpc/php_grpc.c +++ b/src/php/ext/grpc/php_grpc.c @@ -159,6 +159,7 @@ void destroy_grpc_channels() { wrapped_channel.wrapper = le->channel; grpc_channel_wrapper *channel = wrapped_channel.wrapper; grpc_channel_destroy(channel->wrapped); + channel->wrapped = NULL; PHP_GRPC_HASH_FOREACH_END() } From 41bad78ab375c6f80566c4f76846a88ed54c9705 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CMohan?= Date: Fri, 21 Aug 2020 16:36:04 -0700 Subject: [PATCH 21/62] add features and linkstatic to grpc_cc_binary in bazel --- bazel/grpc_build_system.bzl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl index 61b20683754..7ed433fedb7 100644 --- a/bazel/grpc_build_system.bzl +++ b/bazel/grpc_build_system.bzl @@ -231,7 +231,7 @@ def grpc_cc_test(name, srcs = [], deps = [], external_deps = [], args = [], data **args ) -def grpc_cc_binary(name, srcs = [], deps = [], external_deps = [], args = [], data = [], language = "C++", testonly = False, linkshared = False, linkopts = [], tags = []): +def grpc_cc_binary(name, srcs = [], deps = [], external_deps = [], args = [], data = [], language = "C++", testonly = False, linkshared = False, linkstatic = False, linkopts = [], tags = [], features = []): copts = [] if language.upper() == "C": copts = ["-std=c99"] @@ -242,10 +242,12 @@ def grpc_cc_binary(name, srcs = [], deps = [], external_deps = [], args = [], da data = data, testonly = testonly, linkshared = linkshared, + linkstatic = linkstatic, deps = deps + _get_external_deps(external_deps), copts = copts, linkopts = if_not_windows(["-pthread"]) + linkopts, tags = tags, + features = features, ) def grpc_generate_one_off_targets(): From 6088a5e0275685195fa41d0944cf2e256438664d Mon Sep 17 00:00:00 2001 From: Donna Dionne Date: Sat, 22 Aug 2020 13:21:35 -0700 Subject: [PATCH 22/62] Fixing the transient failure that occurs during the case where (Porting --- src/core/ext/filters/client_channel/xds/xds_client.cc | 7 +++++++ test/cpp/end2end/xds_end2end_test.cc | 1 + 2 files changed, 8 insertions(+) diff --git a/src/core/ext/filters/client_channel/xds/xds_client.cc b/src/core/ext/filters/client_channel/xds/xds_client.cc index 3010c6b23d0..46877baef82 100644 --- a/src/core/ext/filters/client_channel/xds/xds_client.cc +++ b/src/core/ext/filters/client_channel/xds/xds_client.cc @@ -1034,6 +1034,13 @@ void XdsClient::ChannelState::AdsCallState::AcceptCdsUpdate( const std::string& cluster_name = p.first; if (cds_update_map.find(cluster_name) == cds_update_map.end()) { ClusterState& cluster_state = xds_client()->cluster_map_[cluster_name]; + // If the resource was newly requested but has not yet been received, + // we don't want to generate an error for the watchers, because this CDS + // response may be in reaction to an earlier request that did not yet + // request the new resource, so its absence from the response does not + // necessarily indicate that the resource does not exist. + // For that case, we rely on the request timeout instead. + if (!cluster_state.update.has_value()) continue; cluster_state.update.reset(); for (const auto& p : cluster_state.watchers) { p.first->OnResourceDoesNotExist(); diff --git a/test/cpp/end2end/xds_end2end_test.cc b/test/cpp/end2end/xds_end2end_test.cc index 7a244e696ae..f6b90541935 100644 --- a/test/cpp/end2end/xds_end2end_test.cc +++ b/test/cpp/end2end/xds_end2end_test.cc @@ -1356,6 +1356,7 @@ class XdsEnd2endTest : public ::testing::TestWithParam { "Performed %d warm up requests against the backends. " "%d succeeded, %d failed, %d dropped.", num_total, num_ok, num_failure, num_drops); + EXPECT_EQ(num_failure, 0); return std::make_tuple(num_ok, num_failure, num_drops); } From 08e3c311d05950107799e1f75f10c356da8ed106 Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Sun, 23 Aug 2020 00:07:08 -0700 Subject: [PATCH 23/62] Merge pull request #23950 from ericgribkoff/directly_query_health xds tests: check health service in wait_for_healthy_backends --- tools/run_tests/run_xds_tests.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/tools/run_tests/run_xds_tests.py b/tools/run_tests/run_xds_tests.py index 14f7cab1364..2d199dc4513 100755 --- a/tools/run_tests/run_xds_tests.py +++ b/tools/run_tests/run_xds_tests.py @@ -957,6 +957,12 @@ def test_header_matching(gcp, original_backend_service, instance_group, set_validate_for_proxyless(gcp, True) +def get_serving_status(instance, service_port): + with grpc.insecure_channel('%s:%d' % (instance, service_port)) as channel: + health_stub = health_pb2_grpc.HealthStub(channel) + return health_stub.Check(health_pb2.HealthCheckRequest()) + + def set_serving_status(instances, service_port, serving): logger.info('setting %s serving status to %s', instances, serving) for instance in instances: @@ -964,16 +970,13 @@ def set_serving_status(instances, service_port, serving): (instance, service_port)) as channel: logger.info('setting %s serving status to %s', instance, serving) stub = test_pb2_grpc.XdsUpdateHealthServiceStub(channel) - health_stub = health_pb2_grpc.HealthStub(channel) - retry_count = 5 for i in range(5): if serving: stub.SetServing(empty_pb2.Empty()) else: stub.SetNotServing(empty_pb2.Empty()) - serving_status = health_stub.Check( - health_pb2.HealthCheckRequest()) + serving_status = get_serving_status(instance, service_port) logger.info('got instance service status %s', serving_status) want_status = health_pb2.HealthCheckResponse.SERVING if serving else health_pb2.HealthCheckResponse.NOT_SERVING if serving_status.status == want_status: @@ -1541,14 +1544,23 @@ def wait_for_healthy_backends(gcp, timeout_sec=_WAIT_FOR_BACKEND_SEC): start_time = time.time() config = {'group': instance_group.url} - expected_size = len(get_instance_names(gcp, instance_group)) + instance_names = get_instance_names(gcp, instance_group) + expected_size = len(instance_names) while time.time() - start_time <= timeout_sec: + for instance_name in instance_names: + try: + status = get_serving_status(instance_name, gcp.service_port) + logger.info('serving status response from %s: %s', + instance_name, status) + except grpc.RpcError as rpc_error: + logger.info('checking serving status of %s failed: %s', + instance_name, rpc_error) result = gcp.compute.backendServices().getHealth( project=gcp.project, backendService=backend_service.name, body=config).execute(num_retries=_GCP_API_RETRIES) if 'healthStatus' in result: - logger.info('received healthStatus: %s', result['healthStatus']) + logger.info('received GCP healthStatus: %s', result['healthStatus']) healthy = True for instance in result['healthStatus']: if instance['healthState'] != 'HEALTHY': @@ -1556,7 +1568,9 @@ def wait_for_healthy_backends(gcp, break if healthy and expected_size == len(result['healthStatus']): return - time.sleep(2) + else: + logger.info('no healthStatus received from GCP') + time.sleep(5) raise Exception('Not all backends became healthy within %d seconds: %s' % (timeout_sec, result)) From f96c83e4c963cf02ef498c5fa5dbb1d201c40b0c Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Sun, 23 Aug 2020 16:47:17 -0700 Subject: [PATCH 24/62] Don't call grpc_call_cancel() from within core. --- .../ext/filters/client_channel/lb_policy/grpclb/grpclb.cc | 2 +- src/core/ext/xds/xds_client.cc | 4 ++-- tools/run_tests/sanity/core_banned_functions.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc index 5dfb43146d9..005d913fb2c 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc @@ -801,7 +801,7 @@ void GrpcLb::BalancerCallState::Orphan() { // lb_on_balancer_status_received_ will complete the cancellation and clean // up. Otherwise, we are here because grpclb_policy has to orphan a failed // call, then the following cancellation will be a no-op. - grpc_call_cancel(lb_call_, nullptr); + grpc_call_cancel_internal(lb_call_); if (client_load_report_timer_callback_pending_) { grpc_timer_cancel(&client_load_report_timer_); } diff --git a/src/core/ext/xds/xds_client.cc b/src/core/ext/xds/xds_client.cc index 67ee5fd2177..061b4c06005 100644 --- a/src/core/ext/xds/xds_client.cc +++ b/src/core/ext/xds/xds_client.cc @@ -787,7 +787,7 @@ void XdsClient::ChannelState::AdsCallState::Orphan() { // on_status_received_ will complete the cancellation and clean up. Otherwise, // we are here because xds_client has to orphan a failed call, then the // following cancellation will be a no-op. - grpc_call_cancel(call_, nullptr); + grpc_call_cancel_internal(call_); state_map_.clear(); // Note that the initial ref is hold by on_status_received_. So the // corresponding unref happens in on_status_received_ instead of here. @@ -1553,7 +1553,7 @@ void XdsClient::ChannelState::LrsCallState::Orphan() { // on_status_received_ will complete the cancellation and clean up. Otherwise, // we are here because xds_client has to orphan a failed call, then the // following cancellation will be a no-op. - grpc_call_cancel(call_, nullptr); + grpc_call_cancel_internal(call_); // Note that the initial ref is hold by on_status_received_. So the // corresponding unref happens in on_status_received_ instead of here. } diff --git a/tools/run_tests/sanity/core_banned_functions.py b/tools/run_tests/sanity/core_banned_functions.py index 934c4089715..a551c801dfd 100755 --- a/tools/run_tests/sanity/core_banned_functions.py +++ b/tools/run_tests/sanity/core_banned_functions.py @@ -43,7 +43,7 @@ BANNED_EXCEPT = { 'grpc_wsa_error(': ['src/core/lib/iomgr/error.cc'], 'grpc_log_if_error(': ['src/core/lib/iomgr/error.cc'], 'grpc_slice_malloc(': ['src/core/lib/slice/slice.cc'], - 'grpc_call_cancel_internal(': ['src/core/lib/surface/call.cc'], + 'grpc_call_cancel(': ['src/core/lib/surface/call.cc'], 'grpc_closure_create(': ['src/core/lib/iomgr/closure.cc'], 'grpc_closure_init(': ['src/core/lib/iomgr/closure.cc'], 'grpc_closure_sched(': ['src/core/lib/iomgr/closure.cc'], From 1263c8014f84c9ebade68ff15ca55b8c56576b81 Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Mon, 24 Aug 2020 12:40:43 -0700 Subject: [PATCH 25/62] Merge pull request #23956 from ericgribkoff/unique_suffix xds test: retry with random suffix after conflict --- tools/run_tests/run_xds_tests.py | 40 +++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/tools/run_tests/run_xds_tests.py b/tools/run_tests/run_xds_tests.py index 2d199dc4513..e5052fdb8c1 100755 --- a/tools/run_tests/run_xds_tests.py +++ b/tools/run_tests/run_xds_tests.py @@ -1662,18 +1662,33 @@ else: try: gcp = GcpState(compute, alpha_compute, args.project_id) - health_check_name = _BASE_HEALTH_CHECK_NAME + args.gcp_suffix - firewall_name = _BASE_FIREWALL_RULE_NAME + args.gcp_suffix - backend_service_name = _BASE_BACKEND_SERVICE_NAME + args.gcp_suffix - alternate_backend_service_name = _BASE_BACKEND_SERVICE_NAME + '-alternate' + args.gcp_suffix - url_map_name = _BASE_URL_MAP_NAME + args.gcp_suffix - service_host_name = _BASE_SERVICE_HOST + args.gcp_suffix - target_proxy_name = _BASE_TARGET_PROXY_NAME + args.gcp_suffix - forwarding_rule_name = _BASE_FORWARDING_RULE_NAME + args.gcp_suffix - template_name = _BASE_TEMPLATE_NAME + args.gcp_suffix - instance_group_name = _BASE_INSTANCE_GROUP_NAME + args.gcp_suffix - same_zone_instance_group_name = _BASE_INSTANCE_GROUP_NAME + '-same-zone' + args.gcp_suffix - secondary_zone_instance_group_name = _BASE_INSTANCE_GROUP_NAME + '-secondary-zone' + args.gcp_suffix + gcp_suffix = args.gcp_suffix + health_check_name = _BASE_HEALTH_CHECK_NAME + gcp_suffix + if not args.use_existing_gcp_resources: + num_attempts = 5 + for i in range(num_attempts): + try: + logger.info('Using GCP suffix %s', gcp_suffix) + create_health_check(gcp, health_check_name) + break + except googleapiclient.errors.HttpError as http_error: + gcp_suffix = '%s-%04d' % (gcp_suffix, random.randint(0, 9999)) + health_check_name = _BASE_HEALTH_CHECK_NAME + gcp_suffix + logger.exception('HttpError when creating health check') + if gcp.health_check is None: + raise Exception('Failed to create health check name after %d ' + 'attempts' % num_attempts) + firewall_name = _BASE_FIREWALL_RULE_NAME + gcp_suffix + backend_service_name = _BASE_BACKEND_SERVICE_NAME + gcp_suffix + alternate_backend_service_name = _BASE_BACKEND_SERVICE_NAME + '-alternate' + gcp_suffix + url_map_name = _BASE_URL_MAP_NAME + gcp_suffix + service_host_name = _BASE_SERVICE_HOST + gcp_suffix + target_proxy_name = _BASE_TARGET_PROXY_NAME + gcp_suffix + forwarding_rule_name = _BASE_FORWARDING_RULE_NAME + gcp_suffix + template_name = _BASE_TEMPLATE_NAME + gcp_suffix + instance_group_name = _BASE_INSTANCE_GROUP_NAME + gcp_suffix + same_zone_instance_group_name = _BASE_INSTANCE_GROUP_NAME + '-same-zone' + gcp_suffix + secondary_zone_instance_group_name = _BASE_INSTANCE_GROUP_NAME + '-secondary-zone' + gcp_suffix if args.use_existing_gcp_resources: logger.info('Reusing existing GCP resources') get_health_check(gcp, health_check_name) @@ -1697,7 +1712,6 @@ try: secondary_zone_instance_group = get_instance_group( gcp, args.secondary_zone, secondary_zone_instance_group_name) else: - create_health_check(gcp, health_check_name) create_health_check_firewall_rule(gcp, firewall_name) backend_service = add_backend_service(gcp, backend_service_name) alternate_backend_service = add_backend_service( From 91b698d232e899d4017a7be6a6ced554b1140f2d Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Tue, 25 Aug 2020 07:03:28 -0700 Subject: [PATCH 26/62] Revert "Merge pull request #23969 from grpc/revert-23918-xds_virtual_host_validation" This reverts commit 13b5f0b62596160cbcc6ab059b32f05d80a41d0b, reversing changes made to 9ea2d7c89dffa7fc3e05d1361eff93f4750572e5. --- .../lb_policy/xds/xds_routing.cc | 88 ++-- .../resolver/xds/xds_resolver.cc | 64 ++- src/core/ext/xds/xds_api.cc | 425 ++++++++++-------- src/core/ext/xds/xds_api.h | 167 +++---- src/core/ext/xds/xds_client.cc | 44 +- src/core/ext/xds/xds_client.h | 2 +- test/cpp/end2end/xds_end2end_test.cc | 12 +- 7 files changed, 409 insertions(+), 393 deletions(-) diff --git a/src/core/ext/filters/client_channel/lb_policy/xds/xds_routing.cc b/src/core/ext/filters/client_channel/lb_policy/xds/xds_routing.cc index 883dcfa55ad..22de026aefa 100644 --- a/src/core/ext/filters/client_channel/lb_policy/xds/xds_routing.cc +++ b/src/core/ext/filters/client_channel/lb_policy/xds/xds_routing.cc @@ -58,7 +58,7 @@ constexpr char kXdsRouting[] = "xds_routing_experimental"; class XdsRoutingLbConfig : public LoadBalancingPolicy::Config { public: struct Route { - XdsApi::RdsUpdate::RdsRoute::Matchers matchers; + XdsApi::Route::Matchers matchers; std::string action; }; using RouteTable = std::vector; @@ -112,7 +112,7 @@ class XdsRoutingLb : public LoadBalancingPolicy { class RoutePicker : public SubchannelPicker { public: struct Route { - const XdsApi::RdsUpdate::RdsRoute::Matchers* matchers; + const XdsApi::Route::Matchers* matchers; RefCountedPtr picker; }; @@ -128,7 +128,7 @@ class XdsRoutingLb : public LoadBalancingPolicy { private: RouteTable route_table_; // Take a reference to config so that we can use - // XdsApi::RdsUpdate::RdsRoute::Matchers from it. + // XdsApi::Route::Matchers from it. RefCountedPtr config_; }; @@ -222,18 +222,14 @@ class XdsRoutingLb : public LoadBalancingPolicy { // XdsRoutingLb::RoutePicker // -bool PathMatch( - const absl::string_view& path, - const XdsApi::RdsUpdate::RdsRoute::Matchers::PathMatcher& path_matcher) { +bool PathMatch(const absl::string_view& path, + const XdsApi::Route::Matchers::PathMatcher& path_matcher) { switch (path_matcher.type) { - case XdsApi::RdsUpdate::RdsRoute::Matchers::PathMatcher::PathMatcherType:: - PREFIX: + case XdsApi::Route::Matchers::PathMatcher::PathMatcherType::PREFIX: return absl::StartsWith(path, path_matcher.string_matcher); - case XdsApi::RdsUpdate::RdsRoute::Matchers::PathMatcher::PathMatcherType:: - PATH: + case XdsApi::Route::Matchers::PathMatcher::PathMatcherType::PATH: return path == path_matcher.string_matcher; - case XdsApi::RdsUpdate::RdsRoute::Matchers::PathMatcher::PathMatcherType:: - REGEX: + case XdsApi::Route::Matchers::PathMatcher::PathMatcherType::REGEX: return RE2::FullMatch(path.data(), *path_matcher.regex_matcher); default: return false; @@ -262,7 +258,7 @@ absl::optional GetMetadataValue( } bool HeaderMatchHelper( - const XdsApi::RdsUpdate::RdsRoute::Matchers::HeaderMatcher& header_matcher, + const XdsApi::Route::Matchers::HeaderMatcher& header_matcher, LoadBalancingPolicy::MetadataInterface* initial_metadata) { std::string concatenated_value; absl::optional value; @@ -279,8 +275,8 @@ bool HeaderMatchHelper( &concatenated_value); } if (!value.has_value()) { - if (header_matcher.type == XdsApi::RdsUpdate::RdsRoute::Matchers:: - HeaderMatcher::HeaderMatcherType::PRESENT) { + if (header_matcher.type == + XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcherType::PRESENT) { return !header_matcher.present_match; } else { // For all other header matcher types, we need the header value to @@ -289,25 +285,20 @@ bool HeaderMatchHelper( } } switch (header_matcher.type) { - case XdsApi::RdsUpdate::RdsRoute::Matchers::HeaderMatcher:: - HeaderMatcherType::EXACT: + case XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcherType::EXACT: return value.value() == header_matcher.string_matcher; - case XdsApi::RdsUpdate::RdsRoute::Matchers::HeaderMatcher:: - HeaderMatcherType::REGEX: + case XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcherType::REGEX: return RE2::FullMatch(value.value().data(), *header_matcher.regex_match); - case XdsApi::RdsUpdate::RdsRoute::Matchers::HeaderMatcher:: - HeaderMatcherType::RANGE: + case XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcherType::RANGE: int64_t int_value; if (!absl::SimpleAtoi(value.value(), &int_value)) { return false; } return int_value >= header_matcher.range_start && int_value < header_matcher.range_end; - case XdsApi::RdsUpdate::RdsRoute::Matchers::HeaderMatcher:: - HeaderMatcherType::PREFIX: + case XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcherType::PREFIX: return absl::StartsWith(value.value(), header_matcher.string_matcher); - case XdsApi::RdsUpdate::RdsRoute::Matchers::HeaderMatcher:: - HeaderMatcherType::SUFFIX: + case XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcherType::SUFFIX: return absl::EndsWith(value.value(), header_matcher.string_matcher); default: return false; @@ -315,8 +306,7 @@ bool HeaderMatchHelper( } bool HeadersMatch( - const std::vector& - header_matchers, + const std::vector& header_matchers, LoadBalancingPolicy::MetadataInterface* initial_metadata) { for (const auto& header_matcher : header_matchers) { bool match = HeaderMatchHelper(header_matcher, initial_metadata); @@ -865,8 +855,8 @@ class XdsRoutingLbFactory : public LoadBalancingPolicyFactory { "field:prefix error: should be string")); } else { path_matcher_seen = true; - route->matchers.path_matcher.type = XdsApi::RdsUpdate::RdsRoute:: - Matchers::PathMatcher::PathMatcherType::PREFIX; + route->matchers.path_matcher.type = + XdsApi::Route::Matchers::PathMatcher::PathMatcherType::PREFIX; route->matchers.path_matcher.string_matcher = it->second.string_value(); } } @@ -881,8 +871,8 @@ class XdsRoutingLbFactory : public LoadBalancingPolicyFactory { error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( "field:path error: should be string")); } else { - route->matchers.path_matcher.type = XdsApi::RdsUpdate::RdsRoute:: - Matchers::PathMatcher::PathMatcherType::PATH; + route->matchers.path_matcher.type = + XdsApi::Route::Matchers::PathMatcher::PathMatcherType::PATH; route->matchers.path_matcher.string_matcher = it->second.string_value(); } @@ -899,8 +889,8 @@ class XdsRoutingLbFactory : public LoadBalancingPolicyFactory { error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( "field:regex error: should be string")); } else { - route->matchers.path_matcher.type = XdsApi::RdsUpdate::RdsRoute:: - Matchers::PathMatcher::PathMatcherType::REGEX; + route->matchers.path_matcher.type = + XdsApi::Route::Matchers::PathMatcher::PathMatcherType::REGEX; route->matchers.path_matcher.regex_matcher = absl::make_unique(it->second.string_value()); } @@ -925,8 +915,8 @@ class XdsRoutingLbFactory : public LoadBalancingPolicyFactory { "value should be of type object")); } else { route->matchers.header_matchers.emplace_back(); - XdsApi::RdsUpdate::RdsRoute::Matchers::HeaderMatcher& - header_matcher = route->matchers.header_matchers.back(); + XdsApi::Route::Matchers::HeaderMatcher& header_matcher = + route->matchers.header_matchers.back(); auto header_it = header_json.object_value().find("name"); if (header_it == header_json.object_value().end()) { error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( @@ -960,8 +950,8 @@ class XdsRoutingLbFactory : public LoadBalancingPolicyFactory { error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( "field:exact_match error: should be string")); } else { - header_matcher.type = XdsApi::RdsUpdate::RdsRoute::Matchers:: - HeaderMatcher::HeaderMatcherType::EXACT; + header_matcher.type = XdsApi::Route::Matchers::HeaderMatcher:: + HeaderMatcherType::EXACT; header_matcher.string_matcher = header_it->second.string_value(); } @@ -978,8 +968,8 @@ class XdsRoutingLbFactory : public LoadBalancingPolicyFactory { error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( "field:regex_match error: should be string")); } else { - header_matcher.type = XdsApi::RdsUpdate::RdsRoute::Matchers:: - HeaderMatcher::HeaderMatcherType::REGEX; + header_matcher.type = XdsApi::Route::Matchers::HeaderMatcher:: + HeaderMatcherType::REGEX; header_matcher.regex_match = absl::make_unique(header_it->second.string_value()); } @@ -1025,8 +1015,8 @@ class XdsRoutingLbFactory : public LoadBalancingPolicyFactory { "field:end missing")); } if (header_matcher.range_end > header_matcher.range_start) { - header_matcher.type = XdsApi::RdsUpdate::RdsRoute:: - Matchers::HeaderMatcher::HeaderMatcherType::RANGE; + header_matcher.type = XdsApi::Route::Matchers:: + HeaderMatcher::HeaderMatcherType::RANGE; } } } @@ -1040,12 +1030,12 @@ class XdsRoutingLbFactory : public LoadBalancingPolicyFactory { } else { header_matcher_seen = true; if (header_it->second.type() == Json::Type::JSON_TRUE) { - header_matcher.type = XdsApi::RdsUpdate::RdsRoute::Matchers:: - HeaderMatcher::HeaderMatcherType::PRESENT; + header_matcher.type = XdsApi::Route::Matchers::HeaderMatcher:: + HeaderMatcherType::PRESENT; header_matcher.present_match = true; } else if (header_it->second.type() == Json::Type::JSON_FALSE) { - header_matcher.type = XdsApi::RdsUpdate::RdsRoute::Matchers:: - HeaderMatcher::HeaderMatcherType::PRESENT; + header_matcher.type = XdsApi::Route::Matchers::HeaderMatcher:: + HeaderMatcherType::PRESENT; header_matcher.present_match = false; } else { error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( @@ -1065,8 +1055,8 @@ class XdsRoutingLbFactory : public LoadBalancingPolicyFactory { error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( "field:prefix_match error: should be string")); } else { - header_matcher.type = XdsApi::RdsUpdate::RdsRoute::Matchers:: - HeaderMatcher::HeaderMatcherType::PREFIX; + header_matcher.type = XdsApi::Route::Matchers::HeaderMatcher:: + HeaderMatcherType::PREFIX; header_matcher.string_matcher = header_it->second.string_value(); } @@ -1084,8 +1074,8 @@ class XdsRoutingLbFactory : public LoadBalancingPolicyFactory { error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( "field:suffix_match error: should be string")); } else { - header_matcher.type = XdsApi::RdsUpdate::RdsRoute::Matchers:: - HeaderMatcher::HeaderMatcherType::SUFFIX; + header_matcher.type = XdsApi::Route::Matchers::HeaderMatcher:: + HeaderMatcherType::SUFFIX; header_matcher.string_matcher = header_it->second.string_value(); } diff --git a/src/core/ext/filters/client_channel/resolver/xds/xds_resolver.cc b/src/core/ext/filters/client_channel/resolver/xds/xds_resolver.cc index c0b93dc69ed..d522ce3a77a 100644 --- a/src/core/ext/filters/client_channel/resolver/xds/xds_resolver.cc +++ b/src/core/ext/filters/client_channel/resolver/xds/xds_resolver.cc @@ -74,7 +74,7 @@ class XdsResolver : public Resolver { public: explicit ListenerWatcher(RefCountedPtr resolver) : resolver_(std::move(resolver)) {} - void OnListenerChanged(XdsApi::LdsUpdate listener_data) override; + void OnListenerChanged(std::vector routes) override; void OnError(grpc_error* error) override; void OnResourceDoesNotExist() override; @@ -92,15 +92,14 @@ class XdsResolver : public Resolver { // Returns the weighted_clusters action name to use from // weighted_cluster_index_map_ for a WeightedClusters route action. std::string WeightedClustersActionName( - const std::vector& - weighted_clusters); + const std::vector& weighted_clusters); // Updates weighted_cluster_index_map_ that will // determine the names of the WeightedCluster actions for the current update. - void UpdateWeightedClusterIndexMap(const XdsApi::RdsUpdate& rds_update); + void UpdateWeightedClusterIndexMap(const std::vector& routes); - // Create the service config generated by the RdsUpdate. - grpc_error* CreateServiceConfig(const XdsApi::RdsUpdate& rds_update, + // Create the service config generated by the list of routes. + grpc_error* CreateServiceConfig(const std::vector& routes, RefCountedPtr* service_config); std::string server_name_; @@ -131,15 +130,15 @@ class XdsResolver : public Resolver { // void XdsResolver::ListenerWatcher::OnListenerChanged( - XdsApi::LdsUpdate listener_data) { + std::vector routes) { if (resolver_->xds_client_ == nullptr) return; if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_resolver_trace)) { gpr_log(GPR_INFO, "[xds_resolver %p] received updated listener data", resolver_.get()); } Result result; - grpc_error* error = resolver_->CreateServiceConfig(*listener_data.rds_update, - &result.service_config); + grpc_error* error = + resolver_->CreateServiceConfig(routes, &result.service_config); if (error != GRPC_ERROR_NONE) { OnError(error); return; @@ -214,23 +213,20 @@ std::string CreateServiceConfigActionCluster(const std::string& cluster_name) { } std::string CreateServiceConfigRoute(const std::string& action_name, - const XdsApi::RdsUpdate::RdsRoute& route) { + const XdsApi::Route& route) { std::vector headers; for (const auto& header : route.matchers.header_matchers) { std::string header_matcher; switch (header.type) { - case XdsApi::RdsUpdate::RdsRoute::Matchers::HeaderMatcher:: - HeaderMatcherType::EXACT: + case XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcherType::EXACT: header_matcher = absl::StrFormat(" \"exact_match\": \"%s\"", header.string_matcher); break; - case XdsApi::RdsUpdate::RdsRoute::Matchers::HeaderMatcher:: - HeaderMatcherType::REGEX: + case XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcherType::REGEX: header_matcher = absl::StrFormat(" \"regex_match\": \"%s\"", header.regex_match->pattern()); break; - case XdsApi::RdsUpdate::RdsRoute::Matchers::HeaderMatcher:: - HeaderMatcherType::RANGE: + case XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcherType::RANGE: header_matcher = absl::StrFormat( " \"range_match\":{\n" " \"start\":%d,\n" @@ -238,19 +234,16 @@ std::string CreateServiceConfigRoute(const std::string& action_name, " }", header.range_start, header.range_end); break; - case XdsApi::RdsUpdate::RdsRoute::Matchers::HeaderMatcher:: - HeaderMatcherType::PRESENT: + case XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcherType::PRESENT: header_matcher = absl::StrFormat(" \"present_match\": %s", header.present_match ? "true" : "false"); break; - case XdsApi::RdsUpdate::RdsRoute::Matchers::HeaderMatcher:: - HeaderMatcherType::PREFIX: + case XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcherType::PREFIX: header_matcher = absl::StrFormat( " \"prefix_match\": \"%s\"", header.string_matcher); break; - case XdsApi::RdsUpdate::RdsRoute::Matchers::HeaderMatcher:: - HeaderMatcherType::SUFFIX: + case XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcherType::SUFFIX: header_matcher = absl::StrFormat( " \"suffix_match\": \"%s\"", header.string_matcher); break; @@ -281,18 +274,15 @@ std::string CreateServiceConfigRoute(const std::string& action_name, } std::string path_match_str; switch (route.matchers.path_matcher.type) { - case XdsApi::RdsUpdate::RdsRoute::Matchers::PathMatcher::PathMatcherType:: - PREFIX: + case XdsApi::Route::Matchers::PathMatcher::PathMatcherType::PREFIX: path_match_str = absl::StrFormat( "\"prefix\": \"%s\",\n", route.matchers.path_matcher.string_matcher); break; - case XdsApi::RdsUpdate::RdsRoute::Matchers::PathMatcher::PathMatcherType:: - PATH: + case XdsApi::Route::Matchers::PathMatcher::PathMatcherType::PATH: path_match_str = absl::StrFormat( "\"path\": \"%s\",\n", route.matchers.path_matcher.string_matcher); break; - case XdsApi::RdsUpdate::RdsRoute::Matchers::PathMatcher::PathMatcherType:: - REGEX: + case XdsApi::Route::Matchers::PathMatcher::PathMatcherType::REGEX: path_match_str = absl::StrFormat("\"regex\": \"%s\",\n", route.matchers.path_matcher.regex_matcher->pattern()); @@ -316,7 +306,7 @@ std::string CreateServiceConfigRoute(const std::string& action_name, // Create the service config for one weighted cluster. std::string CreateServiceConfigActionWeightedCluster( const std::string& name, - const std::vector& clusters) { + const std::vector& clusters) { std::vector config_parts; config_parts.push_back( absl::StrFormat(" \"weighted:%s\":{\n" @@ -354,8 +344,7 @@ struct WeightedClustersKeys { // Returns the cluster names and weights key or the cluster names only key. WeightedClustersKeys GetWeightedClustersKey( - const std::vector& - weighted_clusters) { + const std::vector& weighted_clusters) { std::set cluster_names; std::set cluster_weights; for (const auto& cluster_weight : weighted_clusters) { @@ -368,8 +357,7 @@ WeightedClustersKeys GetWeightedClustersKey( } std::string XdsResolver::WeightedClustersActionName( - const std::vector& - weighted_clusters) { + const std::vector& weighted_clusters) { WeightedClustersKeys keys = GetWeightedClustersKey(weighted_clusters); auto cluster_names_map_it = weighted_cluster_index_map_.find(keys.cluster_names_key); @@ -384,13 +372,13 @@ std::string XdsResolver::WeightedClustersActionName( } void XdsResolver::UpdateWeightedClusterIndexMap( - const XdsApi::RdsUpdate& rds_update) { + const std::vector& routes) { // Construct a list of unique WeightedCluster // actions which we need to process: to find action names std::map actions_to_process; - for (const auto& route : rds_update.routes) { + for (const auto& route : routes) { if (!route.weighted_clusters.empty()) { WeightedClustersKeys keys = GetWeightedClustersKey(route.weighted_clusters); @@ -467,13 +455,13 @@ void XdsResolver::UpdateWeightedClusterIndexMap( } grpc_error* XdsResolver::CreateServiceConfig( - const XdsApi::RdsUpdate& rds_update, + const std::vector& routes, RefCountedPtr* service_config) { - UpdateWeightedClusterIndexMap(rds_update); + UpdateWeightedClusterIndexMap(routes); std::vector actions_vector; std::vector route_table; std::set actions_set; - for (const auto& route : rds_update.routes) { + for (const auto& route : routes) { const std::string action_name = route.weighted_clusters.empty() ? route.cluster_name diff --git a/src/core/ext/xds/xds_api.cc b/src/core/ext/xds/xds_api.cc index 45f5a76f07a..5c21e580167 100644 --- a/src/core/ext/xds/xds_api.cc +++ b/src/core/ext/xds/xds_api.cc @@ -74,11 +74,10 @@ namespace grpc_core { // -// XdsApi::RdsUpdate::RdsRoute::Matchers::PathMatcher +// XdsApi::Route::Matchers::PathMatcher // -XdsApi::RdsUpdate::RdsRoute::Matchers::PathMatcher::PathMatcher( - const PathMatcher& other) +XdsApi::Route::Matchers::PathMatcher::PathMatcher(const PathMatcher& other) : type(other.type) { if (type == PathMatcherType::REGEX) { regex_matcher = absl::make_unique(other.regex_matcher->pattern()); @@ -87,9 +86,8 @@ XdsApi::RdsUpdate::RdsRoute::Matchers::PathMatcher::PathMatcher( } } -XdsApi::RdsUpdate::RdsRoute::Matchers::PathMatcher& -XdsApi::RdsUpdate::RdsRoute::Matchers::PathMatcher::operator=( - const PathMatcher& other) { +XdsApi::Route::Matchers::PathMatcher& XdsApi::Route::Matchers::PathMatcher:: +operator=(const PathMatcher& other) { type = other.type; if (type == PathMatcherType::REGEX) { regex_matcher = absl::make_unique(other.regex_matcher->pattern()); @@ -99,7 +97,7 @@ XdsApi::RdsUpdate::RdsRoute::Matchers::PathMatcher::operator=( return *this; } -bool XdsApi::RdsUpdate::RdsRoute::Matchers::PathMatcher::operator==( +bool XdsApi::Route::Matchers::PathMatcher::operator==( const PathMatcher& other) const { if (type != other.type) return false; if (type == PathMatcherType::REGEX) { @@ -112,8 +110,7 @@ bool XdsApi::RdsUpdate::RdsRoute::Matchers::PathMatcher::operator==( return string_matcher == other.string_matcher; } -std::string XdsApi::RdsUpdate::RdsRoute::Matchers::PathMatcher::ToString() - const { +std::string XdsApi::Route::Matchers::PathMatcher::ToString() const { std::string path_type_string; switch (type) { case PathMatcherType::PATH: @@ -135,10 +132,10 @@ std::string XdsApi::RdsUpdate::RdsRoute::Matchers::PathMatcher::ToString() } // -// XdsApi::RdsUpdate::RdsRoute::Matchers::HeaderMatcher +// XdsApi::Route::Matchers::HeaderMatcher // -XdsApi::RdsUpdate::RdsRoute::Matchers::HeaderMatcher::HeaderMatcher( +XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcher( const HeaderMatcher& other) : name(other.name), type(other.type), invert_match(other.invert_match) { switch (type) { @@ -157,9 +154,8 @@ XdsApi::RdsUpdate::RdsRoute::Matchers::HeaderMatcher::HeaderMatcher( } } -XdsApi::RdsUpdate::RdsRoute::Matchers::HeaderMatcher& -XdsApi::RdsUpdate::RdsRoute::Matchers::HeaderMatcher::operator=( - const HeaderMatcher& other) { +XdsApi::Route::Matchers::HeaderMatcher& XdsApi::Route::Matchers::HeaderMatcher:: +operator=(const HeaderMatcher& other) { name = other.name; type = other.type; invert_match = other.invert_match; @@ -180,7 +176,7 @@ XdsApi::RdsUpdate::RdsRoute::Matchers::HeaderMatcher::operator=( return *this; } -bool XdsApi::RdsUpdate::RdsRoute::Matchers::HeaderMatcher::operator==( +bool XdsApi::Route::Matchers::HeaderMatcher::operator==( const HeaderMatcher& other) const { if (name != other.name) return false; if (type != other.type) return false; @@ -197,8 +193,7 @@ bool XdsApi::RdsUpdate::RdsRoute::Matchers::HeaderMatcher::operator==( } } -std::string XdsApi::RdsUpdate::RdsRoute::Matchers::HeaderMatcher::ToString() - const { +std::string XdsApi::Route::Matchers::HeaderMatcher::ToString() const { switch (type) { case HeaderMatcherType::EXACT: return absl::StrFormat("Header exact match:%s %s:%s", @@ -226,7 +221,11 @@ std::string XdsApi::RdsUpdate::RdsRoute::Matchers::HeaderMatcher::ToString() } } -std::string XdsApi::RdsUpdate::RdsRoute::Matchers::ToString() const { +// +// XdsApi::Route +// + +std::string XdsApi::Route::Matchers::ToString() const { std::vector contents; contents.push_back(path_matcher.ToString()); for (const auto& header_it : header_matchers) { @@ -239,11 +238,11 @@ std::string XdsApi::RdsUpdate::RdsRoute::Matchers::ToString() const { return absl::StrJoin(contents, "\n"); } -std::string XdsApi::RdsUpdate::RdsRoute::ClusterWeight::ToString() const { +std::string XdsApi::Route::ClusterWeight::ToString() const { return absl::StrFormat("{cluster=%s, weight=%d}", name, weight); } -std::string XdsApi::RdsUpdate::RdsRoute::ToString() const { +std::string XdsApi::Route::ToString() const { std::vector contents; contents.push_back(matchers.ToString()); if (!cluster_name.empty()) { @@ -255,12 +254,124 @@ std::string XdsApi::RdsUpdate::RdsRoute::ToString() const { return absl::StrJoin(contents, "\n"); } +// +// XdsApi::RdsUpdate +// + std::string XdsApi::RdsUpdate::ToString() const { - std::vector contents; - for (const auto& route_it : routes) { - contents.push_back(route_it.ToString()); + std::vector vhosts; + for (const VirtualHost& vhost : virtual_hosts) { + vhosts.push_back( + absl::StrCat("vhost={\n" + " domains=[", + absl::StrJoin(vhost.domains, ", "), + "]\n" + " routes=[\n")); + for (const XdsApi::Route& route : vhost.routes) { + vhosts.push_back(" {\n"); + vhosts.push_back(route.ToString()); + vhosts.push_back("\n }\n"); + } + vhosts.push_back(" ]\n"); + vhosts.push_back("]\n"); } - return absl::StrJoin(contents, ",\n"); + return absl::StrJoin(vhosts, ""); +} + +namespace { + +// Better match type has smaller value. +enum MatchType { + EXACT_MATCH, + SUFFIX_MATCH, + PREFIX_MATCH, + UNIVERSE_MATCH, + INVALID_MATCH, +}; + +// Returns true if match succeeds. +bool DomainMatch(MatchType match_type, std::string domain_pattern, + std::string expected_host_name) { + // Normalize the args to lower-case. Domain matching is case-insensitive. + std::transform(domain_pattern.begin(), domain_pattern.end(), + domain_pattern.begin(), + [](unsigned char c) { return std::tolower(c); }); + std::transform(expected_host_name.begin(), expected_host_name.end(), + expected_host_name.begin(), + [](unsigned char c) { return std::tolower(c); }); + if (match_type == EXACT_MATCH) { + return domain_pattern == expected_host_name; + } else if (match_type == SUFFIX_MATCH) { + // Asterisk must match at least one char. + if (expected_host_name.size() < domain_pattern.size()) return false; + absl::string_view pattern_suffix(domain_pattern.c_str() + 1); + absl::string_view host_suffix(expected_host_name.c_str() + + expected_host_name.size() - + pattern_suffix.size()); + return pattern_suffix == host_suffix; + } else if (match_type == PREFIX_MATCH) { + // Asterisk must match at least one char. + if (expected_host_name.size() < domain_pattern.size()) return false; + absl::string_view pattern_prefix(domain_pattern.c_str(), + domain_pattern.size() - 1); + absl::string_view host_prefix(expected_host_name.c_str(), + pattern_prefix.size()); + return pattern_prefix == host_prefix; + } else { + return match_type == UNIVERSE_MATCH; + } +} + +MatchType DomainPatternMatchType(const std::string& domain_pattern) { + if (domain_pattern.empty()) return INVALID_MATCH; + if (domain_pattern.find('*') == std::string::npos) return EXACT_MATCH; + if (domain_pattern == "*") return UNIVERSE_MATCH; + if (domain_pattern[0] == '*') return SUFFIX_MATCH; + if (domain_pattern[domain_pattern.size() - 1] == '*') return PREFIX_MATCH; + return INVALID_MATCH; +} + +} // namespace + +const XdsApi::RdsUpdate::VirtualHost* +XdsApi::RdsUpdate::FindVirtualHostForDomain(const std::string& domain) const { + // Find the best matched virtual host. + // The search order for 4 groups of domain patterns: + // 1. Exact match. + // 2. Suffix match (e.g., "*ABC"). + // 3. Prefix match (e.g., "ABC*"). + // 4. Universe match (i.e., "*"). + // Within each group, longest match wins. + // If the same best matched domain pattern appears in multiple virtual hosts, + // the first matched virtual host wins. + const VirtualHost* target_vhost = nullptr; + MatchType best_match_type = INVALID_MATCH; + size_t longest_match = 0; + // Check each domain pattern in each virtual host to determine the best + // matched virtual host. + for (const VirtualHost& vhost : virtual_hosts) { + for (const std::string& domain_pattern : vhost.domains) { + // Check the match type first. Skip the pattern if it's not better than + // current match. + const MatchType match_type = DomainPatternMatchType(domain_pattern); + // This should be caught by RouteConfigParse(). + GPR_ASSERT(match_type != INVALID_MATCH); + if (match_type > best_match_type) continue; + if (match_type == best_match_type && + domain_pattern.size() <= longest_match) { + continue; + } + // Skip if match fails. + if (!DomainMatch(match_type, domain_pattern, domain)) continue; + // Choose this match. + target_vhost = &vhost; + best_match_type = match_type; + longest_match = domain_pattern.size(); + if (best_match_type == EXACT_MATCH) break; + } + if (best_match_type == EXACT_MATCH) break; + } + return target_vhost; } // @@ -1163,60 +1274,8 @@ void MaybeLogClusterLoadAssignment( } } -// Better match type has smaller value. -enum MatchType { - EXACT_MATCH, - SUFFIX_MATCH, - PREFIX_MATCH, - UNIVERSE_MATCH, - INVALID_MATCH, -}; - -// Returns true if match succeeds. -bool DomainMatch(MatchType match_type, std::string domain_pattern, - std::string expected_host_name) { - // Normalize the args to lower-case. Domain matching is case-insensitive. - std::transform(domain_pattern.begin(), domain_pattern.end(), - domain_pattern.begin(), - [](unsigned char c) { return std::tolower(c); }); - std::transform(expected_host_name.begin(), expected_host_name.end(), - expected_host_name.begin(), - [](unsigned char c) { return std::tolower(c); }); - if (match_type == EXACT_MATCH) { - return domain_pattern == expected_host_name; - } else if (match_type == SUFFIX_MATCH) { - // Asterisk must match at least one char. - if (expected_host_name.size() < domain_pattern.size()) return false; - absl::string_view pattern_suffix(domain_pattern.c_str() + 1); - absl::string_view host_suffix(expected_host_name.c_str() + - expected_host_name.size() - - pattern_suffix.size()); - return pattern_suffix == host_suffix; - } else if (match_type == PREFIX_MATCH) { - // Asterisk must match at least one char. - if (expected_host_name.size() < domain_pattern.size()) return false; - absl::string_view pattern_prefix(domain_pattern.c_str(), - domain_pattern.size() - 1); - absl::string_view host_prefix(expected_host_name.c_str(), - pattern_prefix.size()); - return pattern_prefix == host_prefix; - } else { - return match_type == UNIVERSE_MATCH; - } -} - -MatchType DomainPatternMatchType(const std::string& domain_pattern) { - if (domain_pattern.empty()) return INVALID_MATCH; - if (domain_pattern.find('*') == std::string::npos) return EXACT_MATCH; - if (domain_pattern == "*") return UNIVERSE_MATCH; - if (domain_pattern[0] == '*') return SUFFIX_MATCH; - if (domain_pattern[domain_pattern.size() - 1] == '*') return PREFIX_MATCH; - return INVALID_MATCH; -} - grpc_error* RoutePathMatchParse(const envoy_config_route_v3_RouteMatch* match, - XdsApi::RdsUpdate::RdsRoute* rds_route, - bool* ignore_route) { + XdsApi::Route* route, bool* ignore_route) { if (envoy_config_route_v3_RouteMatch_has_prefix(match)) { absl::string_view prefix = UpbStringToAbsl(envoy_config_route_v3_RouteMatch_prefix(match)); @@ -1241,9 +1300,9 @@ grpc_error* RoutePathMatchParse(const envoy_config_route_v3_RouteMatch* match, return GRPC_ERROR_NONE; } } - rds_route->matchers.path_matcher.type = XdsApi::RdsUpdate::RdsRoute:: - Matchers::PathMatcher::PathMatcherType::PREFIX; - rds_route->matchers.path_matcher.string_matcher = std::string(prefix); + route->matchers.path_matcher.type = + XdsApi::Route::Matchers::PathMatcher::PathMatcherType::PREFIX; + route->matchers.path_matcher.string_matcher = std::string(prefix); } else if (envoy_config_route_v3_RouteMatch_has_path(match)) { absl::string_view path = UpbStringToAbsl(envoy_config_route_v3_RouteMatch_path(match)); @@ -1276,9 +1335,9 @@ grpc_error* RoutePathMatchParse(const envoy_config_route_v3_RouteMatch* match, *ignore_route = true; return GRPC_ERROR_NONE; } - rds_route->matchers.path_matcher.type = XdsApi::RdsUpdate::RdsRoute:: - Matchers::PathMatcher::PathMatcherType::PATH; - rds_route->matchers.path_matcher.string_matcher = std::string(path); + route->matchers.path_matcher.type = + XdsApi::Route::Matchers::PathMatcher::PathMatcherType::PATH; + route->matchers.path_matcher.string_matcher = std::string(path); } else if (envoy_config_route_v3_RouteMatch_has_safe_regex(match)) { const envoy_type_matcher_v3_RegexMatcher* regex_matcher = envoy_config_route_v3_RouteMatch_safe_regex(match); @@ -1290,9 +1349,9 @@ grpc_error* RoutePathMatchParse(const envoy_config_route_v3_RouteMatch* match, return GRPC_ERROR_CREATE_FROM_STATIC_STRING( "Invalid regex string specified in path matcher."); } - rds_route->matchers.path_matcher.type = XdsApi::RdsUpdate::RdsRoute:: - Matchers::PathMatcher::PathMatcherType::REGEX; - rds_route->matchers.path_matcher.regex_matcher = std::move(regex); + route->matchers.path_matcher.type = + XdsApi::Route::Matchers::PathMatcher::PathMatcherType::REGEX; + route->matchers.path_matcher.regex_matcher = std::move(regex); } else { return GRPC_ERROR_CREATE_FROM_STATIC_STRING( "Invalid route path specifier specified."); @@ -1301,19 +1360,18 @@ grpc_error* RoutePathMatchParse(const envoy_config_route_v3_RouteMatch* match, } grpc_error* RouteHeaderMatchersParse( - const envoy_config_route_v3_RouteMatch* match, - XdsApi::RdsUpdate::RdsRoute* rds_route) { + const envoy_config_route_v3_RouteMatch* match, XdsApi::Route* route) { size_t size; const envoy_config_route_v3_HeaderMatcher* const* headers = envoy_config_route_v3_RouteMatch_headers(match, &size); for (size_t i = 0; i < size; ++i) { const envoy_config_route_v3_HeaderMatcher* header = headers[i]; - XdsApi::RdsUpdate::RdsRoute::Matchers::HeaderMatcher header_matcher; + XdsApi::Route::Matchers::HeaderMatcher header_matcher; header_matcher.name = UpbStringToStdString(envoy_config_route_v3_HeaderMatcher_name(header)); if (envoy_config_route_v3_HeaderMatcher_has_exact_match(header)) { - header_matcher.type = XdsApi::RdsUpdate::RdsRoute::Matchers:: - HeaderMatcher::HeaderMatcherType::EXACT; + header_matcher.type = + XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcherType::EXACT; header_matcher.string_matcher = UpbStringToStdString( envoy_config_route_v3_HeaderMatcher_exact_match(header)); } else if (envoy_config_route_v3_HeaderMatcher_has_safe_regex_match( @@ -1328,12 +1386,12 @@ grpc_error* RouteHeaderMatchersParse( return GRPC_ERROR_CREATE_FROM_STATIC_STRING( "Invalid regex string specified in header matcher."); } - header_matcher.type = XdsApi::RdsUpdate::RdsRoute::Matchers:: - HeaderMatcher::HeaderMatcherType::REGEX; + header_matcher.type = + XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcherType::REGEX; header_matcher.regex_match = std::move(regex); } else if (envoy_config_route_v3_HeaderMatcher_has_range_match(header)) { - header_matcher.type = XdsApi::RdsUpdate::RdsRoute::Matchers:: - HeaderMatcher::HeaderMatcherType::RANGE; + header_matcher.type = + XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcherType::RANGE; const envoy_type_v3_Int64Range* range_matcher = envoy_config_route_v3_HeaderMatcher_range_match(header); header_matcher.range_start = @@ -1345,18 +1403,18 @@ grpc_error* RouteHeaderMatchersParse( "cannot be smaller than start."); } } else if (envoy_config_route_v3_HeaderMatcher_has_present_match(header)) { - header_matcher.type = XdsApi::RdsUpdate::RdsRoute::Matchers:: - HeaderMatcher::HeaderMatcherType::PRESENT; + header_matcher.type = + XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcherType::PRESENT; header_matcher.present_match = envoy_config_route_v3_HeaderMatcher_present_match(header); } else if (envoy_config_route_v3_HeaderMatcher_has_prefix_match(header)) { - header_matcher.type = XdsApi::RdsUpdate::RdsRoute::Matchers:: - HeaderMatcher::HeaderMatcherType::PREFIX; + header_matcher.type = + XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcherType::PREFIX; header_matcher.string_matcher = UpbStringToStdString( envoy_config_route_v3_HeaderMatcher_prefix_match(header)); } else if (envoy_config_route_v3_HeaderMatcher_has_suffix_match(header)) { - header_matcher.type = XdsApi::RdsUpdate::RdsRoute::Matchers:: - HeaderMatcher::HeaderMatcherType::SUFFIX; + header_matcher.type = + XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcherType::SUFFIX; header_matcher.string_matcher = UpbStringToStdString( envoy_config_route_v3_HeaderMatcher_suffix_match(header)); } else { @@ -1365,14 +1423,13 @@ grpc_error* RouteHeaderMatchersParse( } header_matcher.invert_match = envoy_config_route_v3_HeaderMatcher_invert_match(header); - rds_route->matchers.header_matchers.emplace_back(std::move(header_matcher)); + route->matchers.header_matchers.emplace_back(std::move(header_matcher)); } return GRPC_ERROR_NONE; } grpc_error* RouteRuntimeFractionParse( - const envoy_config_route_v3_RouteMatch* match, - XdsApi::RdsUpdate::RdsRoute* rds_route) { + const envoy_config_route_v3_RouteMatch* match, XdsApi::Route* route) { const envoy_config_core_v3_RuntimeFractionalPercent* runtime_fraction = envoy_config_route_v3_RouteMatch_runtime_fraction(match); if (runtime_fraction != nullptr) { @@ -1398,26 +1455,25 @@ grpc_error* RouteRuntimeFractionParse( return GRPC_ERROR_CREATE_FROM_STATIC_STRING( "Unknown denominator type"); } - rds_route->matchers.fraction_per_million = numerator; + route->matchers.fraction_per_million = numerator; } } return GRPC_ERROR_NONE; } -grpc_error* RouteActionParse(const envoy_config_route_v3_Route* route, - XdsApi::RdsUpdate::RdsRoute* rds_route, - bool* ignore_route) { - if (!envoy_config_route_v3_Route_has_route(route)) { +grpc_error* RouteActionParse(const envoy_config_route_v3_Route* route_msg, + XdsApi::Route* route, bool* ignore_route) { + if (!envoy_config_route_v3_Route_has_route(route_msg)) { return GRPC_ERROR_CREATE_FROM_STATIC_STRING( "No RouteAction found in route."); } const envoy_config_route_v3_RouteAction* route_action = - envoy_config_route_v3_Route_route(route); + envoy_config_route_v3_Route_route(route_msg); // Get the cluster or weighted_clusters in the RouteAction. if (envoy_config_route_v3_RouteAction_has_cluster(route_action)) { - rds_route->cluster_name = UpbStringToStdString( + route->cluster_name = UpbStringToStdString( envoy_config_route_v3_RouteAction_cluster(route_action)); - if (rds_route->cluster_name.size() == 0) { + if (route->cluster_name.size() == 0) { return GRPC_ERROR_CREATE_FROM_STATIC_STRING( "RouteAction cluster contains empty cluster name."); } @@ -1439,7 +1495,7 @@ grpc_error* RouteActionParse(const envoy_config_route_v3_Route* route, for (size_t j = 0; j < clusters_size; ++j) { const envoy_config_route_v3_WeightedCluster_ClusterWeight* cluster_weight = clusters[j]; - XdsApi::RdsUpdate::RdsRoute::ClusterWeight cluster; + XdsApi::Route::ClusterWeight cluster; cluster.name = UpbStringToStdString( envoy_config_route_v3_WeightedCluster_ClusterWeight_name( cluster_weight)); @@ -1457,13 +1513,13 @@ grpc_error* RouteActionParse(const envoy_config_route_v3_Route* route, } cluster.weight = google_protobuf_UInt32Value_value(weight); sum_of_weights += cluster.weight; - rds_route->weighted_clusters.emplace_back(std::move(cluster)); + route->weighted_clusters.emplace_back(std::move(cluster)); } if (total_weight != sum_of_weights) { return GRPC_ERROR_CREATE_FROM_STATIC_STRING( "RouteAction weighted_cluster has incorrect total weight"); } - if (rds_route->weighted_clusters.empty()) { + if (route->weighted_clusters.empty()) { return GRPC_ERROR_CREATE_FROM_STATIC_STRING( "RouteAction weighted_cluster has no valid clusters specified."); } @@ -1478,101 +1534,73 @@ grpc_error* RouteActionParse(const envoy_config_route_v3_Route* route, grpc_error* RouteConfigParse( XdsClient* client, TraceFlag* tracer, const envoy_config_route_v3_RouteConfiguration* route_config, - const std::string& expected_server_name, XdsApi::RdsUpdate* rds_update) { + XdsApi::RdsUpdate* rds_update) { MaybeLogRouteConfiguration(client, tracer, route_config); // Get the virtual hosts. size_t size; const envoy_config_route_v3_VirtualHost* const* virtual_hosts = envoy_config_route_v3_RouteConfiguration_virtual_hosts(route_config, &size); - // Find the best matched virtual host. - // The search order for 4 groups of domain patterns: - // 1. Exact match. - // 2. Suffix match (e.g., "*ABC"). - // 3. Prefix match (e.g., "ABC*"). - // 4. Universe match (i.e., "*"). - // Within each group, longest match wins. - // If the same best matched domain pattern appears in multiple virtual hosts, - // the first matched virtual host wins. - const envoy_config_route_v3_VirtualHost* target_virtual_host = nullptr; - MatchType best_match_type = INVALID_MATCH; - size_t longest_match = 0; - // Check each domain pattern in each virtual host to determine the best - // matched virtual host. for (size_t i = 0; i < size; ++i) { + rds_update->virtual_hosts.emplace_back(); + XdsApi::RdsUpdate::VirtualHost& vhost = rds_update->virtual_hosts.back(); + // Parse domains. size_t domain_size; upb_strview const* domains = envoy_config_route_v3_VirtualHost_domains( virtual_hosts[i], &domain_size); for (size_t j = 0; j < domain_size; ++j) { - const std::string domain_pattern(domains[j].data, domains[j].size); - // Check the match type first. Skip the pattern if it's not better than - // current match. + std::string domain_pattern = UpbStringToStdString(domains[j]); const MatchType match_type = DomainPatternMatchType(domain_pattern); if (match_type == INVALID_MATCH) { return GRPC_ERROR_CREATE_FROM_STATIC_STRING("Invalid domain pattern."); } - if (match_type > best_match_type) continue; - if (match_type == best_match_type && - domain_pattern.size() <= longest_match) { + vhost.domains.emplace_back(std::move(domain_pattern)); + } + if (vhost.domains.empty()) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING("VirtualHost has no domains"); + } + // Parse routes. + size_t num_routes; + const envoy_config_route_v3_Route* const* routes = + envoy_config_route_v3_VirtualHost_routes(virtual_hosts[i], &num_routes); + if (num_routes < 1) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "No route found in the virtual host."); + } + // Loop over the whole list of routes + for (size_t j = 0; j < num_routes; ++j) { + const envoy_config_route_v3_RouteMatch* match = + envoy_config_route_v3_Route_match(routes[j]); + size_t query_parameters_size; + static_cast(envoy_config_route_v3_RouteMatch_query_parameters( + match, &query_parameters_size)); + if (query_parameters_size > 0) { continue; } - // Skip if match fails. - if (!DomainMatch(match_type, domain_pattern, expected_server_name)) { - continue; + XdsApi::Route route; + bool ignore_route = false; + grpc_error* error = RoutePathMatchParse(match, &route, &ignore_route); + if (error != GRPC_ERROR_NONE) return error; + if (ignore_route) continue; + error = RouteHeaderMatchersParse(match, &route); + if (error != GRPC_ERROR_NONE) return error; + error = RouteRuntimeFractionParse(match, &route); + if (error != GRPC_ERROR_NONE) return error; + error = RouteActionParse(routes[j], &route, &ignore_route); + if (error != GRPC_ERROR_NONE) return error; + if (ignore_route) continue; + const google_protobuf_BoolValue* case_sensitive = + envoy_config_route_v3_RouteMatch_case_sensitive(match); + if (case_sensitive != nullptr && + !google_protobuf_BoolValue_value(case_sensitive)) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "case_sensitive if set must be set to true."); } - // Choose this match. - target_virtual_host = virtual_hosts[i]; - best_match_type = match_type; - longest_match = domain_pattern.size(); - if (best_match_type == EXACT_MATCH) break; - } - if (best_match_type == EXACT_MATCH) break; - } - if (target_virtual_host == nullptr) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "No matched virtual host found in the route config."); - } - // Get the route list from the matched virtual host. - const envoy_config_route_v3_Route* const* routes = - envoy_config_route_v3_VirtualHost_routes(target_virtual_host, &size); - if (size < 1) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "No route found in the virtual host."); - } - // Loop over the whole list of routes - for (size_t i = 0; i < size; ++i) { - const envoy_config_route_v3_Route* route = routes[i]; - const envoy_config_route_v3_RouteMatch* match = - envoy_config_route_v3_Route_match(route); - size_t query_parameters_size; - static_cast(envoy_config_route_v3_RouteMatch_query_parameters( - match, &query_parameters_size)); - if (query_parameters_size > 0) { - continue; + vhost.routes.emplace_back(std::move(route)); } - XdsApi::RdsUpdate::RdsRoute rds_route; - bool ignore_route = false; - grpc_error* error = RoutePathMatchParse(match, &rds_route, &ignore_route); - if (error != GRPC_ERROR_NONE) return error; - if (ignore_route) continue; - error = RouteHeaderMatchersParse(match, &rds_route); - if (error != GRPC_ERROR_NONE) return error; - error = RouteRuntimeFractionParse(match, &rds_route); - if (error != GRPC_ERROR_NONE) return error; - error = RouteActionParse(route, &rds_route, &ignore_route); - if (error != GRPC_ERROR_NONE) return error; - if (ignore_route) continue; - const google_protobuf_BoolValue* case_sensitive = - envoy_config_route_v3_RouteMatch_case_sensitive(match); - if (case_sensitive != nullptr && - !google_protobuf_BoolValue_value(case_sensitive)) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "case_sensitive if set must be set to true."); + if (vhost.routes.empty()) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING("No valid routes specified."); } - rds_update->routes.emplace_back(std::move(rds_route)); - } - if (rds_update->routes.empty()) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING("No valid routes specified."); } return GRPC_ERROR_NONE; } @@ -1630,11 +1658,11 @@ grpc_error* LdsResponseParse( envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_route_config( http_connection_manager); XdsApi::RdsUpdate rds_update; - grpc_error* error = RouteConfigParse(client, tracer, route_config, - expected_server_name, &rds_update); + grpc_error* error = + RouteConfigParse(client, tracer, route_config, &rds_update); if (error != GRPC_ERROR_NONE) return error; lds_update->emplace(); - (*lds_update)->rds_update.emplace(std::move(rds_update)); + (*lds_update)->rds_update = std::move(rds_update); return GRPC_ERROR_NONE; } // Validate that RDS must be used to get the route_config dynamically. @@ -1671,7 +1699,6 @@ grpc_error* LdsResponseParse( grpc_error* RdsResponseParse( XdsClient* client, TraceFlag* tracer, const envoy_service_discovery_v3_DiscoveryResponse* response, - const std::string& expected_server_name, const std::set& expected_route_configuration_names, absl::optional* rds_update, upb_arena* arena) { // Get the resources from the response. @@ -1703,8 +1730,8 @@ grpc_error* RdsResponseParse( } // Parse the route_config. XdsApi::RdsUpdate local_rds_update; - grpc_error* error = RouteConfigParse( - client, tracer, route_config, expected_server_name, &local_rds_update); + grpc_error* error = + RouteConfigParse(client, tracer, route_config, &local_rds_update); if (error != GRPC_ERROR_NONE) return error; rds_update->emplace(std::move(local_rds_update)); return GRPC_ERROR_NONE; @@ -2043,9 +2070,9 @@ XdsApi::AdsParseResult XdsApi::ParseAdsResponse( LdsResponseParse(client_, tracer_, response, expected_server_name, &result.lds_update, arena.ptr()); } else if (IsRds(result.type_url)) { - result.parse_error = RdsResponseParse( - client_, tracer_, response, expected_server_name, - expected_route_configuration_names, &result.rds_update, arena.ptr()); + result.parse_error = RdsResponseParse(client_, tracer_, response, + expected_route_configuration_names, + &result.rds_update, arena.ptr()); } else if (IsCds(result.type_url)) { result.parse_error = CdsResponseParse(client_, tracer_, response, expected_cluster_names, diff --git a/src/core/ext/xds/xds_api.h b/src/core/ext/xds/xds_api.h index 3ec720e7104..265fec43c09 100644 --- a/src/core/ext/xds/xds_api.h +++ b/src/core/ext/xds/xds_api.h @@ -46,98 +46,109 @@ class XdsApi { static const char* kCdsTypeUrl; static const char* kEdsTypeUrl; - struct RdsUpdate { - // TODO(donnadionne): When we can use absl::variant<>, consider using that - // for: PathMatcher, HeaderMatcher, cluster_name and weighted_clusters - struct RdsRoute { - // Matchers for this route. - struct Matchers { - struct PathMatcher { - enum class PathMatcherType { - PATH, // path stored in string_matcher field - PREFIX, // prefix stored in string_matcher field - REGEX, // regex stored in regex_matcher field - }; - PathMatcherType type; - std::string string_matcher; - std::unique_ptr regex_matcher; - - PathMatcher() = default; - PathMatcher(const PathMatcher& other); - PathMatcher& operator=(const PathMatcher& other); - bool operator==(const PathMatcher& other) const; - std::string ToString() const; - }; - - struct HeaderMatcher { - enum class HeaderMatcherType { - EXACT, // value stored in string_matcher field - REGEX, // uses regex_match field - RANGE, // uses range_start and range_end fields - PRESENT, // uses present_match field - PREFIX, // prefix stored in string_matcher field - SUFFIX, // suffix stored in string_matcher field - }; - std::string name; - HeaderMatcherType type; - int64_t range_start; - int64_t range_end; - std::string string_matcher; - std::unique_ptr regex_match; - bool present_match; - // invert_match field may or may not exisit, so initialize it to - // false. - bool invert_match = false; - - HeaderMatcher() = default; - HeaderMatcher(const HeaderMatcher& other); - HeaderMatcher& operator=(const HeaderMatcher& other); - bool operator==(const HeaderMatcher& other) const; - std::string ToString() const; + // TODO(donnadionne): When we can use absl::variant<>, consider using that + // for: PathMatcher, HeaderMatcher, cluster_name and weighted_clusters + struct Route { + // Matchers for this route. + struct Matchers { + struct PathMatcher { + enum class PathMatcherType { + PATH, // path stored in string_matcher field + PREFIX, // prefix stored in string_matcher field + REGEX, // regex stored in regex_matcher field }; - - PathMatcher path_matcher; - std::vector header_matchers; - absl::optional fraction_per_million; - - bool operator==(const Matchers& other) const { - return (path_matcher == other.path_matcher && - header_matchers == other.header_matchers && - fraction_per_million == other.fraction_per_million); - } + PathMatcherType type; + std::string string_matcher; + std::unique_ptr regex_matcher; + + PathMatcher() = default; + PathMatcher(const PathMatcher& other); + PathMatcher& operator=(const PathMatcher& other); + bool operator==(const PathMatcher& other) const; std::string ToString() const; }; - Matchers matchers; - - // Action for this route. - // TODO(roth): When we can use absl::variant<>, consider using that - // here, to enforce the fact that only one of the two fields can be set. - std::string cluster_name; - struct ClusterWeight { + struct HeaderMatcher { + enum class HeaderMatcherType { + EXACT, // value stored in string_matcher field + REGEX, // uses regex_match field + RANGE, // uses range_start and range_end fields + PRESENT, // uses present_match field + PREFIX, // prefix stored in string_matcher field + SUFFIX, // suffix stored in string_matcher field + }; std::string name; - uint32_t weight; - bool operator==(const ClusterWeight& other) const { - return (name == other.name && weight == other.weight); - } + HeaderMatcherType type; + int64_t range_start; + int64_t range_end; + std::string string_matcher; + std::unique_ptr regex_match; + bool present_match; + // invert_match field may or may not exisit, so initialize it to + // false. + bool invert_match = false; + + HeaderMatcher() = default; + HeaderMatcher(const HeaderMatcher& other); + HeaderMatcher& operator=(const HeaderMatcher& other); + bool operator==(const HeaderMatcher& other) const; std::string ToString() const; }; - std::vector weighted_clusters; - bool operator==(const RdsRoute& other) const { - return (matchers == other.matchers && - cluster_name == other.cluster_name && - weighted_clusters == other.weighted_clusters); + PathMatcher path_matcher; + std::vector header_matchers; + absl::optional fraction_per_million; + + bool operator==(const Matchers& other) const { + return (path_matcher == other.path_matcher && + header_matchers == other.header_matchers && + fraction_per_million == other.fraction_per_million); } std::string ToString() const; }; - std::vector routes; + Matchers matchers; + + // Action for this route. + // TODO(roth): When we can use absl::variant<>, consider using that + // here, to enforce the fact that only one of the two fields can be set. + std::string cluster_name; + struct ClusterWeight { + std::string name; + uint32_t weight; + bool operator==(const ClusterWeight& other) const { + return (name == other.name && weight == other.weight); + } + std::string ToString() const; + }; + std::vector weighted_clusters; + + bool operator==(const Route& other) const { + return (matchers == other.matchers && + cluster_name == other.cluster_name && + weighted_clusters == other.weighted_clusters); + } + std::string ToString() const; + }; + + struct RdsUpdate { + struct VirtualHost { + std::vector domains; + std::vector routes; + + bool operator==(const VirtualHost& other) const { + return domains == other.domains && routes == other.routes; + } + }; + + std::vector virtual_hosts; bool operator==(const RdsUpdate& other) const { - return routes == other.routes; + return virtual_hosts == other.virtual_hosts; } std::string ToString() const; + const VirtualHost* FindVirtualHostForDomain( + const std::string& domain) const; }; // TODO(roth): When we can use absl::variant<>, consider using that @@ -145,8 +156,8 @@ class XdsApi { struct LdsUpdate { // The name to use in the RDS request. std::string route_config_name; - // The name to use in the CDS request. Present if the LDS response has it - // inlined. + // The RouteConfiguration to use for this listener. + // Present only if it is inlined in the LDS response. absl::optional rds_update; bool operator==(const LdsUpdate& other) const { diff --git a/src/core/ext/xds/xds_client.cc b/src/core/ext/xds/xds_client.cc index 67ee5fd2177..b582a0c5c32 100644 --- a/src/core/ext/xds/xds_client.cc +++ b/src/core/ext/xds/xds_client.cc @@ -892,13 +892,8 @@ void XdsClient::ChannelState::AdsCallState::AcceptLdsUpdate( ? lds_update->route_config_name.c_str() : "")); if (lds_update->rds_update.has_value()) { - gpr_log(GPR_INFO, "RouteConfiguration contains %" PRIuPTR " routes", - lds_update->rds_update.value().routes.size()); - for (size_t i = 0; i < lds_update->rds_update.value().routes.size(); - ++i) { - gpr_log(GPR_INFO, "Route %" PRIuPTR ":\n%s", i, - lds_update->rds_update.value().routes[i].ToString().c_str()); - } + gpr_log(GPR_INFO, "RouteConfiguration: %s", + lds_update->rds_update->ToString().c_str()); } } auto& lds_state = state_map_[XdsApi::kLdsTypeUrl]; @@ -924,8 +919,16 @@ void XdsClient::ChannelState::AdsCallState::AcceptLdsUpdate( if (xds_client()->lds_result_->rds_update.has_value()) { // If the RouteConfiguration was found inlined in LDS response, notify // the watcher immediately. - xds_client()->listener_watcher_->OnListenerChanged( - *xds_client()->lds_result_); + const XdsApi::RdsUpdate::VirtualHost* vhost = + xds_client()->lds_result_->rds_update->FindVirtualHostForDomain( + xds_client()->server_name_); + if (vhost == nullptr) { + xds_client()->listener_watcher_->OnError( + GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "no VirtualHost found for domain")); + } else { + xds_client()->listener_watcher_->OnListenerChanged(vhost->routes); + } } else { // Send RDS request for dynamic resolution. Subscribe(XdsApi::kRdsTypeUrl, @@ -944,14 +947,8 @@ void XdsClient::ChannelState::AdsCallState::AcceptRdsUpdate( return; } if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { - gpr_log(GPR_INFO, - "[xds_client %p] RDS update received; RouteConfiguration contains " - "%" PRIuPTR " routes", - this, rds_update.value().routes.size()); - for (size_t i = 0; i < rds_update.value().routes.size(); ++i) { - gpr_log(GPR_INFO, "Route %" PRIuPTR ":\n%s", i, - rds_update.value().routes[i].ToString().c_str()); - } + gpr_log(GPR_INFO, "[xds_client %p] RDS update received:\n%s", xds_client(), + rds_update->ToString().c_str()); } auto& rds_state = state_map_[XdsApi::kRdsTypeUrl]; auto& state = @@ -969,9 +966,16 @@ void XdsClient::ChannelState::AdsCallState::AcceptRdsUpdate( } xds_client()->rds_result_ = std::move(rds_update); // Notify the watcher. - XdsApi::LdsUpdate lds_result = *xds_client()->lds_result_; - lds_result.rds_update = xds_client()->rds_result_; - xds_client()->listener_watcher_->OnListenerChanged(lds_result); + const XdsApi::RdsUpdate::VirtualHost* vhost = + xds_client()->rds_result_->FindVirtualHostForDomain( + xds_client()->server_name_); + if (vhost == nullptr) { + xds_client()->listener_watcher_->OnError( + GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "no VirtualHost found for domain")); + } else { + xds_client()->listener_watcher_->OnListenerChanged(vhost->routes); + } } void XdsClient::ChannelState::AdsCallState::AcceptCdsUpdate( diff --git a/src/core/ext/xds/xds_client.h b/src/core/ext/xds/xds_client.h index cb8864b0e27..c9184348a8c 100644 --- a/src/core/ext/xds/xds_client.h +++ b/src/core/ext/xds/xds_client.h @@ -45,7 +45,7 @@ class XdsClient : public InternallyRefCounted { public: virtual ~ListenerWatcherInterface() = default; - virtual void OnListenerChanged(XdsApi::LdsUpdate listener_data) = 0; + virtual void OnListenerChanged(std::vector routes) = 0; virtual void OnError(grpc_error* error) = 0; diff --git a/test/cpp/end2end/xds_end2end_test.cc b/test/cpp/end2end/xds_end2end_test.cc index b2099f12806..28820334055 100644 --- a/test/cpp/end2end/xds_end2end_test.cc +++ b/test/cpp/end2end/xds_end2end_test.cc @@ -2500,7 +2500,7 @@ TEST_P(LdsRdsTest, ListenerRemoved) { AdsServiceImpl::ResponseState::ACKED); } -// Tests that LDS client should send a NACK if matching domain can't be found in +// Tests that LDS client ACKS but fails if matching domain can't be found in // the LDS response. TEST_P(LdsRdsTest, NoMatchedDomain) { RouteConfiguration route_config = @@ -2511,10 +2511,10 @@ TEST_P(LdsRdsTest, NoMatchedDomain) { SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); + // Do a bit of polling, to allow the ACK to get to the ADS server. + channel_->WaitForConnected(grpc_timeout_milliseconds_to_deadline(10)); const auto& response_state = RouteConfigurationResponseState(0); - EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); - EXPECT_EQ(response_state.error_message, - "No matched virtual host found in the route config."); + EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::ACKED); } // Tests that LDS client should choose the virtual host with matching domain if @@ -2525,10 +2525,6 @@ TEST_P(LdsRdsTest, ChooseMatchedDomain) { *(route_config.add_virtual_hosts()) = route_config.virtual_hosts(0); route_config.mutable_virtual_hosts(0)->clear_domains(); route_config.mutable_virtual_hosts(0)->add_domains("unmatched_domain"); - route_config.mutable_virtual_hosts(0) - ->mutable_routes(0) - ->mutable_route() - ->mutable_cluster_header(); SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); From 5adf794eec0aa493cc15892638eec840019da4ba Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Tue, 25 Aug 2020 07:18:52 -0700 Subject: [PATCH 27/62] Increase polling timeout for NoMatchedDomain test. --- test/cpp/end2end/xds_end2end_test.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/cpp/end2end/xds_end2end_test.cc b/test/cpp/end2end/xds_end2end_test.cc index 28820334055..b3303c62771 100644 --- a/test/cpp/end2end/xds_end2end_test.cc +++ b/test/cpp/end2end/xds_end2end_test.cc @@ -2500,7 +2500,7 @@ TEST_P(LdsRdsTest, ListenerRemoved) { AdsServiceImpl::ResponseState::ACKED); } -// Tests that LDS client ACKS but fails if matching domain can't be found in +// Tests that LDS client ACKs but fails if matching domain can't be found in // the LDS response. TEST_P(LdsRdsTest, NoMatchedDomain) { RouteConfiguration route_config = @@ -2512,7 +2512,7 @@ TEST_P(LdsRdsTest, NoMatchedDomain) { SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); // Do a bit of polling, to allow the ACK to get to the ADS server. - channel_->WaitForConnected(grpc_timeout_milliseconds_to_deadline(10)); + channel_->WaitForConnected(grpc_timeout_milliseconds_to_deadline(100)); const auto& response_state = RouteConfigurationResponseState(0); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::ACKED); } From 3013a636d224cd36b14dba2d342e0210c83a9548 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 15 Jul 2020 11:37:15 +0200 Subject: [PATCH 28/62] C# generator: use buffer serialization --- src/compiler/csharp_generator.cc | 69 ++++++++++++++++++++++++++++++-- 1 file changed, 66 insertions(+), 3 deletions(-) diff --git a/src/compiler/csharp_generator.cc b/src/compiler/csharp_generator.cc index 8c8ea5e090f..3ec554b5837 100644 --- a/src/compiler/csharp_generator.cc +++ b/src/compiler/csharp_generator.cc @@ -325,13 +325,76 @@ std::vector GetUsedMessages( void GenerateMarshallerFields(Printer* out, const ServiceDescriptor* service) { std::vector used_messages = GetUsedMessages(service); + if (used_messages.size() != 0) { + // Generate static helper methods for serialization/deserialization + out->Print( + "static void __Helper_SerializeMessage(" + "global::Google.Protobuf.IMessage message, " + "grpc::SerializationContext context)\n" + "{\n"); + out->Indent(); + out->Print( + "#if !GRPC_DISABLE_PROTOBUF_BUFFER_SERIALIZATION\n" + "if (message is global::Google.Protobuf.IBufferMessage)\n" + "{\n"); + out->Indent(); + out->Print( + "context.SetPayloadLength(message.CalculateSize());\n" + "global::Google.Protobuf.MessageExtensions.WriteTo(message, " + "context.GetBufferWriter());\n" + "context.Complete();\n" + "return;\n"); + out->Outdent(); + out->Print( + "}\n" + "#endif\n"); + out->Print( + "context.Complete(" + "global::Google.Protobuf.MessageExtensions.ToByteArray(message));\n"); + out->Outdent(); + out->Print("}\n\n"); + + out->Print( + "static class __Helper_MessageCache\n" + "{\n"); + out->Indent(); + out->Print( + "public static readonly bool IsBufferMessage = " + "global::System.Reflection.IntrospectionExtensions.GetTypeInfo(typeof(" + "global::Google.Protobuf.IBufferMessage)).IsAssignableFrom(typeof(T));" + "\n"); + out->Outdent(); + out->Print("}\n\n"); + + out->Print( + "static T __Helper_DeserializeMessage(" + "grpc::DeserializationContext context, " + "global::Google.Protobuf.MessageParser parser) " + "where T : global::Google.Protobuf.IMessage\n" + "{\n"); + out->Indent(); + out->Print( + "#if !GRPC_DISABLE_PROTOBUF_BUFFER_SERIALIZATION\n" + "if (__Helper_MessageCache.IsBufferMessage)\n" + "{\n"); + out->Indent(); + out->Print( + "return parser.ParseFrom(context.PayloadAsReadOnlySequence());\n"); + out->Outdent(); + out->Print( + "}\n" + "#endif\n"); + out->Print("return parser.ParseFrom(context.PayloadAsNewBuffer());\n"); + out->Outdent(); + out->Print("}\n\n"); + } + for (size_t i = 0; i < used_messages.size(); i++) { const Descriptor* message = used_messages[i]; out->Print( "static readonly grpc::Marshaller<$type$> $fieldname$ = " - "grpc::Marshallers.Create((arg) => " - "global::Google.Protobuf.MessageExtensions.ToByteArray(arg), " - "$type$.Parser.ParseFrom);\n", + "grpc::Marshallers.Create(__Helper_SerializeMessage, " + "context => __Helper_DeserializeMessage(context, $type$.Parser));\n", "fieldname", GetMarshallerFieldName(message), "type", GetClassName(message)); } From c1c268fff498a7517b0779191c9c7182e8cd7e5c Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 15 Jul 2020 11:38:39 +0200 Subject: [PATCH 29/62] regenerate C# grpc stubs --- src/csharp/Grpc.Examples/MathGrpc.cs | 38 +++- src/csharp/Grpc.HealthCheck/HealthGrpc.cs | 34 +++- .../BenchmarkServiceGrpc.cs | 34 +++- .../Grpc.IntegrationTesting/MetricsGrpc.cs | 36 +++- .../ReportQpsScenarioServiceGrpc.cs | 34 +++- .../Grpc.IntegrationTesting/TestGrpc.cs | 178 ++++++++++++++++-- .../WorkerServiceGrpc.cs | 44 ++++- src/csharp/Grpc.Reflection/ReflectionGrpc.cs | 34 +++- 8 files changed, 396 insertions(+), 36 deletions(-) diff --git a/src/csharp/Grpc.Examples/MathGrpc.cs b/src/csharp/Grpc.Examples/MathGrpc.cs index 5186d2f6dae..cb7cb089f72 100644 --- a/src/csharp/Grpc.Examples/MathGrpc.cs +++ b/src/csharp/Grpc.Examples/MathGrpc.cs @@ -27,10 +27,40 @@ namespace Math { { static readonly string __ServiceName = "math.Math"; - static readonly grpc::Marshaller __Marshaller_math_DivArgs = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Math.DivArgs.Parser.ParseFrom); - static readonly grpc::Marshaller __Marshaller_math_DivReply = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Math.DivReply.Parser.ParseFrom); - static readonly grpc::Marshaller __Marshaller_math_FibArgs = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Math.FibArgs.Parser.ParseFrom); - static readonly grpc::Marshaller __Marshaller_math_Num = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Math.Num.Parser.ParseFrom); + static void __Helper_SerializeMessage(global::Google.Protobuf.IMessage message, grpc::SerializationContext context) + { + #if !GRPC_DISABLE_PROTOBUF_BUFFER_SERIALIZATION + if (message is global::Google.Protobuf.IBufferMessage) + { + context.SetPayloadLength(message.CalculateSize()); + global::Google.Protobuf.MessageExtensions.WriteTo(message, context.GetBufferWriter()); + context.Complete(); + return; + } + #endif + context.Complete(global::Google.Protobuf.MessageExtensions.ToByteArray(message)); + } + + static class __Helper_MessageCache + { + public static readonly bool IsBufferMessage = global::System.Reflection.IntrospectionExtensions.GetTypeInfo(typeof(global::Google.Protobuf.IBufferMessage)).IsAssignableFrom(typeof(T)); + } + + static T __Helper_DeserializeMessage(grpc::DeserializationContext context, global::Google.Protobuf.MessageParser parser) where T : global::Google.Protobuf.IMessage + { + #if !GRPC_DISABLE_PROTOBUF_BUFFER_SERIALIZATION + if (__Helper_MessageCache.IsBufferMessage) + { + return parser.ParseFrom(context.PayloadAsReadOnlySequence()); + } + #endif + return parser.ParseFrom(context.PayloadAsNewBuffer()); + } + + static readonly grpc::Marshaller __Marshaller_math_DivArgs = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::Math.DivArgs.Parser)); + static readonly grpc::Marshaller __Marshaller_math_DivReply = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::Math.DivReply.Parser)); + static readonly grpc::Marshaller __Marshaller_math_FibArgs = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::Math.FibArgs.Parser)); + static readonly grpc::Marshaller __Marshaller_math_Num = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::Math.Num.Parser)); static readonly grpc::Method __Method_Div = new grpc::Method( grpc::MethodType.Unary, diff --git a/src/csharp/Grpc.HealthCheck/HealthGrpc.cs b/src/csharp/Grpc.HealthCheck/HealthGrpc.cs index d5051f3b05a..ffa666c5155 100644 --- a/src/csharp/Grpc.HealthCheck/HealthGrpc.cs +++ b/src/csharp/Grpc.HealthCheck/HealthGrpc.cs @@ -30,8 +30,38 @@ namespace Grpc.Health.V1 { { static readonly string __ServiceName = "grpc.health.v1.Health"; - static readonly grpc::Marshaller __Marshaller_grpc_health_v1_HealthCheckRequest = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Health.V1.HealthCheckRequest.Parser.ParseFrom); - static readonly grpc::Marshaller __Marshaller_grpc_health_v1_HealthCheckResponse = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Health.V1.HealthCheckResponse.Parser.ParseFrom); + static void __Helper_SerializeMessage(global::Google.Protobuf.IMessage message, grpc::SerializationContext context) + { + #if !GRPC_DISABLE_PROTOBUF_BUFFER_SERIALIZATION + if (message is global::Google.Protobuf.IBufferMessage) + { + context.SetPayloadLength(message.CalculateSize()); + global::Google.Protobuf.MessageExtensions.WriteTo(message, context.GetBufferWriter()); + context.Complete(); + return; + } + #endif + context.Complete(global::Google.Protobuf.MessageExtensions.ToByteArray(message)); + } + + static class __Helper_MessageCache + { + public static readonly bool IsBufferMessage = global::System.Reflection.IntrospectionExtensions.GetTypeInfo(typeof(global::Google.Protobuf.IBufferMessage)).IsAssignableFrom(typeof(T)); + } + + static T __Helper_DeserializeMessage(grpc::DeserializationContext context, global::Google.Protobuf.MessageParser parser) where T : global::Google.Protobuf.IMessage + { + #if !GRPC_DISABLE_PROTOBUF_BUFFER_SERIALIZATION + if (__Helper_MessageCache.IsBufferMessage) + { + return parser.ParseFrom(context.PayloadAsReadOnlySequence()); + } + #endif + return parser.ParseFrom(context.PayloadAsNewBuffer()); + } + + static readonly grpc::Marshaller __Marshaller_grpc_health_v1_HealthCheckRequest = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::Grpc.Health.V1.HealthCheckRequest.Parser)); + static readonly grpc::Marshaller __Marshaller_grpc_health_v1_HealthCheckResponse = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::Grpc.Health.V1.HealthCheckResponse.Parser)); static readonly grpc::Method __Method_Check = new grpc::Method( grpc::MethodType.Unary, diff --git a/src/csharp/Grpc.IntegrationTesting/BenchmarkServiceGrpc.cs b/src/csharp/Grpc.IntegrationTesting/BenchmarkServiceGrpc.cs index 4a608be318c..7c65cda71b5 100644 --- a/src/csharp/Grpc.IntegrationTesting/BenchmarkServiceGrpc.cs +++ b/src/csharp/Grpc.IntegrationTesting/BenchmarkServiceGrpc.cs @@ -29,8 +29,38 @@ namespace Grpc.Testing { { static readonly string __ServiceName = "grpc.testing.BenchmarkService"; - static readonly grpc::Marshaller __Marshaller_grpc_testing_SimpleRequest = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Testing.SimpleRequest.Parser.ParseFrom); - static readonly grpc::Marshaller __Marshaller_grpc_testing_SimpleResponse = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Testing.SimpleResponse.Parser.ParseFrom); + static void __Helper_SerializeMessage(global::Google.Protobuf.IMessage message, grpc::SerializationContext context) + { + #if !GRPC_DISABLE_PROTOBUF_BUFFER_SERIALIZATION + if (message is global::Google.Protobuf.IBufferMessage) + { + context.SetPayloadLength(message.CalculateSize()); + global::Google.Protobuf.MessageExtensions.WriteTo(message, context.GetBufferWriter()); + context.Complete(); + return; + } + #endif + context.Complete(global::Google.Protobuf.MessageExtensions.ToByteArray(message)); + } + + static class __Helper_MessageCache + { + public static readonly bool IsBufferMessage = global::System.Reflection.IntrospectionExtensions.GetTypeInfo(typeof(global::Google.Protobuf.IBufferMessage)).IsAssignableFrom(typeof(T)); + } + + static T __Helper_DeserializeMessage(grpc::DeserializationContext context, global::Google.Protobuf.MessageParser parser) where T : global::Google.Protobuf.IMessage + { + #if !GRPC_DISABLE_PROTOBUF_BUFFER_SERIALIZATION + if (__Helper_MessageCache.IsBufferMessage) + { + return parser.ParseFrom(context.PayloadAsReadOnlySequence()); + } + #endif + return parser.ParseFrom(context.PayloadAsNewBuffer()); + } + + static readonly grpc::Marshaller __Marshaller_grpc_testing_SimpleRequest = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::Grpc.Testing.SimpleRequest.Parser)); + static readonly grpc::Marshaller __Marshaller_grpc_testing_SimpleResponse = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::Grpc.Testing.SimpleResponse.Parser)); static readonly grpc::Method __Method_UnaryCall = new grpc::Method( grpc::MethodType.Unary, diff --git a/src/csharp/Grpc.IntegrationTesting/MetricsGrpc.cs b/src/csharp/Grpc.IntegrationTesting/MetricsGrpc.cs index fbc3690d2ac..0b364cdd783 100644 --- a/src/csharp/Grpc.IntegrationTesting/MetricsGrpc.cs +++ b/src/csharp/Grpc.IntegrationTesting/MetricsGrpc.cs @@ -33,9 +33,39 @@ namespace Grpc.Testing { { static readonly string __ServiceName = "grpc.testing.MetricsService"; - static readonly grpc::Marshaller __Marshaller_grpc_testing_EmptyMessage = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Testing.EmptyMessage.Parser.ParseFrom); - static readonly grpc::Marshaller __Marshaller_grpc_testing_GaugeResponse = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Testing.GaugeResponse.Parser.ParseFrom); - static readonly grpc::Marshaller __Marshaller_grpc_testing_GaugeRequest = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Testing.GaugeRequest.Parser.ParseFrom); + static void __Helper_SerializeMessage(global::Google.Protobuf.IMessage message, grpc::SerializationContext context) + { + #if !GRPC_DISABLE_PROTOBUF_BUFFER_SERIALIZATION + if (message is global::Google.Protobuf.IBufferMessage) + { + context.SetPayloadLength(message.CalculateSize()); + global::Google.Protobuf.MessageExtensions.WriteTo(message, context.GetBufferWriter()); + context.Complete(); + return; + } + #endif + context.Complete(global::Google.Protobuf.MessageExtensions.ToByteArray(message)); + } + + static class __Helper_MessageCache + { + public static readonly bool IsBufferMessage = global::System.Reflection.IntrospectionExtensions.GetTypeInfo(typeof(global::Google.Protobuf.IBufferMessage)).IsAssignableFrom(typeof(T)); + } + + static T __Helper_DeserializeMessage(grpc::DeserializationContext context, global::Google.Protobuf.MessageParser parser) where T : global::Google.Protobuf.IMessage + { + #if !GRPC_DISABLE_PROTOBUF_BUFFER_SERIALIZATION + if (__Helper_MessageCache.IsBufferMessage) + { + return parser.ParseFrom(context.PayloadAsReadOnlySequence()); + } + #endif + return parser.ParseFrom(context.PayloadAsNewBuffer()); + } + + static readonly grpc::Marshaller __Marshaller_grpc_testing_EmptyMessage = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::Grpc.Testing.EmptyMessage.Parser)); + static readonly grpc::Marshaller __Marshaller_grpc_testing_GaugeResponse = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::Grpc.Testing.GaugeResponse.Parser)); + static readonly grpc::Marshaller __Marshaller_grpc_testing_GaugeRequest = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::Grpc.Testing.GaugeRequest.Parser)); static readonly grpc::Method __Method_GetAllGauges = new grpc::Method( grpc::MethodType.ServerStreaming, diff --git a/src/csharp/Grpc.IntegrationTesting/ReportQpsScenarioServiceGrpc.cs b/src/csharp/Grpc.IntegrationTesting/ReportQpsScenarioServiceGrpc.cs index e35c7c39fd5..98f03dbab3a 100644 --- a/src/csharp/Grpc.IntegrationTesting/ReportQpsScenarioServiceGrpc.cs +++ b/src/csharp/Grpc.IntegrationTesting/ReportQpsScenarioServiceGrpc.cs @@ -29,8 +29,38 @@ namespace Grpc.Testing { { static readonly string __ServiceName = "grpc.testing.ReportQpsScenarioService"; - static readonly grpc::Marshaller __Marshaller_grpc_testing_ScenarioResult = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Testing.ScenarioResult.Parser.ParseFrom); - static readonly grpc::Marshaller __Marshaller_grpc_testing_Void = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Testing.Void.Parser.ParseFrom); + static void __Helper_SerializeMessage(global::Google.Protobuf.IMessage message, grpc::SerializationContext context) + { + #if !GRPC_DISABLE_PROTOBUF_BUFFER_SERIALIZATION + if (message is global::Google.Protobuf.IBufferMessage) + { + context.SetPayloadLength(message.CalculateSize()); + global::Google.Protobuf.MessageExtensions.WriteTo(message, context.GetBufferWriter()); + context.Complete(); + return; + } + #endif + context.Complete(global::Google.Protobuf.MessageExtensions.ToByteArray(message)); + } + + static class __Helper_MessageCache + { + public static readonly bool IsBufferMessage = global::System.Reflection.IntrospectionExtensions.GetTypeInfo(typeof(global::Google.Protobuf.IBufferMessage)).IsAssignableFrom(typeof(T)); + } + + static T __Helper_DeserializeMessage(grpc::DeserializationContext context, global::Google.Protobuf.MessageParser parser) where T : global::Google.Protobuf.IMessage + { + #if !GRPC_DISABLE_PROTOBUF_BUFFER_SERIALIZATION + if (__Helper_MessageCache.IsBufferMessage) + { + return parser.ParseFrom(context.PayloadAsReadOnlySequence()); + } + #endif + return parser.ParseFrom(context.PayloadAsNewBuffer()); + } + + static readonly grpc::Marshaller __Marshaller_grpc_testing_ScenarioResult = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::Grpc.Testing.ScenarioResult.Parser)); + static readonly grpc::Marshaller __Marshaller_grpc_testing_Void = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::Grpc.Testing.Void.Parser)); static readonly grpc::Method __Method_ReportScenario = new grpc::Method( grpc::MethodType.Unary, diff --git a/src/csharp/Grpc.IntegrationTesting/TestGrpc.cs b/src/csharp/Grpc.IntegrationTesting/TestGrpc.cs index ffa8388e3fb..168c0892bae 100644 --- a/src/csharp/Grpc.IntegrationTesting/TestGrpc.cs +++ b/src/csharp/Grpc.IntegrationTesting/TestGrpc.cs @@ -34,13 +34,43 @@ namespace Grpc.Testing { { static readonly string __ServiceName = "grpc.testing.TestService"; - static readonly grpc::Marshaller __Marshaller_grpc_testing_Empty = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Testing.Empty.Parser.ParseFrom); - static readonly grpc::Marshaller __Marshaller_grpc_testing_SimpleRequest = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Testing.SimpleRequest.Parser.ParseFrom); - static readonly grpc::Marshaller __Marshaller_grpc_testing_SimpleResponse = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Testing.SimpleResponse.Parser.ParseFrom); - static readonly grpc::Marshaller __Marshaller_grpc_testing_StreamingOutputCallRequest = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Testing.StreamingOutputCallRequest.Parser.ParseFrom); - static readonly grpc::Marshaller __Marshaller_grpc_testing_StreamingOutputCallResponse = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Testing.StreamingOutputCallResponse.Parser.ParseFrom); - static readonly grpc::Marshaller __Marshaller_grpc_testing_StreamingInputCallRequest = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Testing.StreamingInputCallRequest.Parser.ParseFrom); - static readonly grpc::Marshaller __Marshaller_grpc_testing_StreamingInputCallResponse = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Testing.StreamingInputCallResponse.Parser.ParseFrom); + static void __Helper_SerializeMessage(global::Google.Protobuf.IMessage message, grpc::SerializationContext context) + { + #if !GRPC_DISABLE_PROTOBUF_BUFFER_SERIALIZATION + if (message is global::Google.Protobuf.IBufferMessage) + { + context.SetPayloadLength(message.CalculateSize()); + global::Google.Protobuf.MessageExtensions.WriteTo(message, context.GetBufferWriter()); + context.Complete(); + return; + } + #endif + context.Complete(global::Google.Protobuf.MessageExtensions.ToByteArray(message)); + } + + static class __Helper_MessageCache + { + public static readonly bool IsBufferMessage = global::System.Reflection.IntrospectionExtensions.GetTypeInfo(typeof(global::Google.Protobuf.IBufferMessage)).IsAssignableFrom(typeof(T)); + } + + static T __Helper_DeserializeMessage(grpc::DeserializationContext context, global::Google.Protobuf.MessageParser parser) where T : global::Google.Protobuf.IMessage + { + #if !GRPC_DISABLE_PROTOBUF_BUFFER_SERIALIZATION + if (__Helper_MessageCache.IsBufferMessage) + { + return parser.ParseFrom(context.PayloadAsReadOnlySequence()); + } + #endif + return parser.ParseFrom(context.PayloadAsNewBuffer()); + } + + static readonly grpc::Marshaller __Marshaller_grpc_testing_Empty = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::Grpc.Testing.Empty.Parser)); + static readonly grpc::Marshaller __Marshaller_grpc_testing_SimpleRequest = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::Grpc.Testing.SimpleRequest.Parser)); + static readonly grpc::Marshaller __Marshaller_grpc_testing_SimpleResponse = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::Grpc.Testing.SimpleResponse.Parser)); + static readonly grpc::Marshaller __Marshaller_grpc_testing_StreamingOutputCallRequest = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::Grpc.Testing.StreamingOutputCallRequest.Parser)); + static readonly grpc::Marshaller __Marshaller_grpc_testing_StreamingOutputCallResponse = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::Grpc.Testing.StreamingOutputCallResponse.Parser)); + static readonly grpc::Marshaller __Marshaller_grpc_testing_StreamingInputCallRequest = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::Grpc.Testing.StreamingInputCallRequest.Parser)); + static readonly grpc::Marshaller __Marshaller_grpc_testing_StreamingInputCallResponse = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::Grpc.Testing.StreamingInputCallResponse.Parser)); static readonly grpc::Method __Method_EmptyCall = new grpc::Method( grpc::MethodType.Unary, @@ -565,7 +595,37 @@ namespace Grpc.Testing { { static readonly string __ServiceName = "grpc.testing.UnimplementedService"; - static readonly grpc::Marshaller __Marshaller_grpc_testing_Empty = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Testing.Empty.Parser.ParseFrom); + static void __Helper_SerializeMessage(global::Google.Protobuf.IMessage message, grpc::SerializationContext context) + { + #if !GRPC_DISABLE_PROTOBUF_BUFFER_SERIALIZATION + if (message is global::Google.Protobuf.IBufferMessage) + { + context.SetPayloadLength(message.CalculateSize()); + global::Google.Protobuf.MessageExtensions.WriteTo(message, context.GetBufferWriter()); + context.Complete(); + return; + } + #endif + context.Complete(global::Google.Protobuf.MessageExtensions.ToByteArray(message)); + } + + static class __Helper_MessageCache + { + public static readonly bool IsBufferMessage = global::System.Reflection.IntrospectionExtensions.GetTypeInfo(typeof(global::Google.Protobuf.IBufferMessage)).IsAssignableFrom(typeof(T)); + } + + static T __Helper_DeserializeMessage(grpc::DeserializationContext context, global::Google.Protobuf.MessageParser parser) where T : global::Google.Protobuf.IMessage + { + #if !GRPC_DISABLE_PROTOBUF_BUFFER_SERIALIZATION + if (__Helper_MessageCache.IsBufferMessage) + { + return parser.ParseFrom(context.PayloadAsReadOnlySequence()); + } + #endif + return parser.ParseFrom(context.PayloadAsNewBuffer()); + } + + static readonly grpc::Marshaller __Marshaller_grpc_testing_Empty = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::Grpc.Testing.Empty.Parser)); static readonly grpc::Method __Method_UnimplementedCall = new grpc::Method( grpc::MethodType.Unary, @@ -696,9 +756,39 @@ namespace Grpc.Testing { { static readonly string __ServiceName = "grpc.testing.ReconnectService"; - static readonly grpc::Marshaller __Marshaller_grpc_testing_ReconnectParams = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Testing.ReconnectParams.Parser.ParseFrom); - static readonly grpc::Marshaller __Marshaller_grpc_testing_Empty = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Testing.Empty.Parser.ParseFrom); - static readonly grpc::Marshaller __Marshaller_grpc_testing_ReconnectInfo = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Testing.ReconnectInfo.Parser.ParseFrom); + static void __Helper_SerializeMessage(global::Google.Protobuf.IMessage message, grpc::SerializationContext context) + { + #if !GRPC_DISABLE_PROTOBUF_BUFFER_SERIALIZATION + if (message is global::Google.Protobuf.IBufferMessage) + { + context.SetPayloadLength(message.CalculateSize()); + global::Google.Protobuf.MessageExtensions.WriteTo(message, context.GetBufferWriter()); + context.Complete(); + return; + } + #endif + context.Complete(global::Google.Protobuf.MessageExtensions.ToByteArray(message)); + } + + static class __Helper_MessageCache + { + public static readonly bool IsBufferMessage = global::System.Reflection.IntrospectionExtensions.GetTypeInfo(typeof(global::Google.Protobuf.IBufferMessage)).IsAssignableFrom(typeof(T)); + } + + static T __Helper_DeserializeMessage(grpc::DeserializationContext context, global::Google.Protobuf.MessageParser parser) where T : global::Google.Protobuf.IMessage + { + #if !GRPC_DISABLE_PROTOBUF_BUFFER_SERIALIZATION + if (__Helper_MessageCache.IsBufferMessage) + { + return parser.ParseFrom(context.PayloadAsReadOnlySequence()); + } + #endif + return parser.ParseFrom(context.PayloadAsNewBuffer()); + } + + static readonly grpc::Marshaller __Marshaller_grpc_testing_ReconnectParams = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::Grpc.Testing.ReconnectParams.Parser)); + static readonly grpc::Marshaller __Marshaller_grpc_testing_Empty = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::Grpc.Testing.Empty.Parser)); + static readonly grpc::Marshaller __Marshaller_grpc_testing_ReconnectInfo = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::Grpc.Testing.ReconnectInfo.Parser)); static readonly grpc::Method __Method_Start = new grpc::Method( grpc::MethodType.Unary, @@ -825,8 +915,38 @@ namespace Grpc.Testing { { static readonly string __ServiceName = "grpc.testing.LoadBalancerStatsService"; - static readonly grpc::Marshaller __Marshaller_grpc_testing_LoadBalancerStatsRequest = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Testing.LoadBalancerStatsRequest.Parser.ParseFrom); - static readonly grpc::Marshaller __Marshaller_grpc_testing_LoadBalancerStatsResponse = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Testing.LoadBalancerStatsResponse.Parser.ParseFrom); + static void __Helper_SerializeMessage(global::Google.Protobuf.IMessage message, grpc::SerializationContext context) + { + #if !GRPC_DISABLE_PROTOBUF_BUFFER_SERIALIZATION + if (message is global::Google.Protobuf.IBufferMessage) + { + context.SetPayloadLength(message.CalculateSize()); + global::Google.Protobuf.MessageExtensions.WriteTo(message, context.GetBufferWriter()); + context.Complete(); + return; + } + #endif + context.Complete(global::Google.Protobuf.MessageExtensions.ToByteArray(message)); + } + + static class __Helper_MessageCache + { + public static readonly bool IsBufferMessage = global::System.Reflection.IntrospectionExtensions.GetTypeInfo(typeof(global::Google.Protobuf.IBufferMessage)).IsAssignableFrom(typeof(T)); + } + + static T __Helper_DeserializeMessage(grpc::DeserializationContext context, global::Google.Protobuf.MessageParser parser) where T : global::Google.Protobuf.IMessage + { + #if !GRPC_DISABLE_PROTOBUF_BUFFER_SERIALIZATION + if (__Helper_MessageCache.IsBufferMessage) + { + return parser.ParseFrom(context.PayloadAsReadOnlySequence()); + } + #endif + return parser.ParseFrom(context.PayloadAsNewBuffer()); + } + + static readonly grpc::Marshaller __Marshaller_grpc_testing_LoadBalancerStatsRequest = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::Grpc.Testing.LoadBalancerStatsRequest.Parser)); + static readonly grpc::Marshaller __Marshaller_grpc_testing_LoadBalancerStatsResponse = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::Grpc.Testing.LoadBalancerStatsResponse.Parser)); static readonly grpc::Method __Method_GetClientStats = new grpc::Method( grpc::MethodType.Unary, @@ -957,7 +1077,37 @@ namespace Grpc.Testing { { static readonly string __ServiceName = "grpc.testing.XdsUpdateHealthService"; - static readonly grpc::Marshaller __Marshaller_grpc_testing_Empty = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Testing.Empty.Parser.ParseFrom); + static void __Helper_SerializeMessage(global::Google.Protobuf.IMessage message, grpc::SerializationContext context) + { + #if !GRPC_DISABLE_PROTOBUF_BUFFER_SERIALIZATION + if (message is global::Google.Protobuf.IBufferMessage) + { + context.SetPayloadLength(message.CalculateSize()); + global::Google.Protobuf.MessageExtensions.WriteTo(message, context.GetBufferWriter()); + context.Complete(); + return; + } + #endif + context.Complete(global::Google.Protobuf.MessageExtensions.ToByteArray(message)); + } + + static class __Helper_MessageCache + { + public static readonly bool IsBufferMessage = global::System.Reflection.IntrospectionExtensions.GetTypeInfo(typeof(global::Google.Protobuf.IBufferMessage)).IsAssignableFrom(typeof(T)); + } + + static T __Helper_DeserializeMessage(grpc::DeserializationContext context, global::Google.Protobuf.MessageParser parser) where T : global::Google.Protobuf.IMessage + { + #if !GRPC_DISABLE_PROTOBUF_BUFFER_SERIALIZATION + if (__Helper_MessageCache.IsBufferMessage) + { + return parser.ParseFrom(context.PayloadAsReadOnlySequence()); + } + #endif + return parser.ParseFrom(context.PayloadAsNewBuffer()); + } + + static readonly grpc::Marshaller __Marshaller_grpc_testing_Empty = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::Grpc.Testing.Empty.Parser)); static readonly grpc::Method __Method_SetServing = new grpc::Method( grpc::MethodType.Unary, diff --git a/src/csharp/Grpc.IntegrationTesting/WorkerServiceGrpc.cs b/src/csharp/Grpc.IntegrationTesting/WorkerServiceGrpc.cs index 66378081626..31bab7ca424 100644 --- a/src/csharp/Grpc.IntegrationTesting/WorkerServiceGrpc.cs +++ b/src/csharp/Grpc.IntegrationTesting/WorkerServiceGrpc.cs @@ -29,13 +29,43 @@ namespace Grpc.Testing { { static readonly string __ServiceName = "grpc.testing.WorkerService"; - static readonly grpc::Marshaller __Marshaller_grpc_testing_ServerArgs = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Testing.ServerArgs.Parser.ParseFrom); - static readonly grpc::Marshaller __Marshaller_grpc_testing_ServerStatus = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Testing.ServerStatus.Parser.ParseFrom); - static readonly grpc::Marshaller __Marshaller_grpc_testing_ClientArgs = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Testing.ClientArgs.Parser.ParseFrom); - static readonly grpc::Marshaller __Marshaller_grpc_testing_ClientStatus = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Testing.ClientStatus.Parser.ParseFrom); - static readonly grpc::Marshaller __Marshaller_grpc_testing_CoreRequest = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Testing.CoreRequest.Parser.ParseFrom); - static readonly grpc::Marshaller __Marshaller_grpc_testing_CoreResponse = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Testing.CoreResponse.Parser.ParseFrom); - static readonly grpc::Marshaller __Marshaller_grpc_testing_Void = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Testing.Void.Parser.ParseFrom); + static void __Helper_SerializeMessage(global::Google.Protobuf.IMessage message, grpc::SerializationContext context) + { + #if !GRPC_DISABLE_PROTOBUF_BUFFER_SERIALIZATION + if (message is global::Google.Protobuf.IBufferMessage) + { + context.SetPayloadLength(message.CalculateSize()); + global::Google.Protobuf.MessageExtensions.WriteTo(message, context.GetBufferWriter()); + context.Complete(); + return; + } + #endif + context.Complete(global::Google.Protobuf.MessageExtensions.ToByteArray(message)); + } + + static class __Helper_MessageCache + { + public static readonly bool IsBufferMessage = global::System.Reflection.IntrospectionExtensions.GetTypeInfo(typeof(global::Google.Protobuf.IBufferMessage)).IsAssignableFrom(typeof(T)); + } + + static T __Helper_DeserializeMessage(grpc::DeserializationContext context, global::Google.Protobuf.MessageParser parser) where T : global::Google.Protobuf.IMessage + { + #if !GRPC_DISABLE_PROTOBUF_BUFFER_SERIALIZATION + if (__Helper_MessageCache.IsBufferMessage) + { + return parser.ParseFrom(context.PayloadAsReadOnlySequence()); + } + #endif + return parser.ParseFrom(context.PayloadAsNewBuffer()); + } + + static readonly grpc::Marshaller __Marshaller_grpc_testing_ServerArgs = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::Grpc.Testing.ServerArgs.Parser)); + static readonly grpc::Marshaller __Marshaller_grpc_testing_ServerStatus = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::Grpc.Testing.ServerStatus.Parser)); + static readonly grpc::Marshaller __Marshaller_grpc_testing_ClientArgs = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::Grpc.Testing.ClientArgs.Parser)); + static readonly grpc::Marshaller __Marshaller_grpc_testing_ClientStatus = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::Grpc.Testing.ClientStatus.Parser)); + static readonly grpc::Marshaller __Marshaller_grpc_testing_CoreRequest = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::Grpc.Testing.CoreRequest.Parser)); + static readonly grpc::Marshaller __Marshaller_grpc_testing_CoreResponse = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::Grpc.Testing.CoreResponse.Parser)); + static readonly grpc::Marshaller __Marshaller_grpc_testing_Void = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::Grpc.Testing.Void.Parser)); static readonly grpc::Method __Method_RunServer = new grpc::Method( grpc::MethodType.DuplexStreaming, diff --git a/src/csharp/Grpc.Reflection/ReflectionGrpc.cs b/src/csharp/Grpc.Reflection/ReflectionGrpc.cs index fa482eba494..d5822be354c 100644 --- a/src/csharp/Grpc.Reflection/ReflectionGrpc.cs +++ b/src/csharp/Grpc.Reflection/ReflectionGrpc.cs @@ -29,8 +29,38 @@ namespace Grpc.Reflection.V1Alpha { { static readonly string __ServiceName = "grpc.reflection.v1alpha.ServerReflection"; - static readonly grpc::Marshaller __Marshaller_grpc_reflection_v1alpha_ServerReflectionRequest = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Reflection.V1Alpha.ServerReflectionRequest.Parser.ParseFrom); - static readonly grpc::Marshaller __Marshaller_grpc_reflection_v1alpha_ServerReflectionResponse = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Grpc.Reflection.V1Alpha.ServerReflectionResponse.Parser.ParseFrom); + static void __Helper_SerializeMessage(global::Google.Protobuf.IMessage message, grpc::SerializationContext context) + { + #if !GRPC_DISABLE_PROTOBUF_BUFFER_SERIALIZATION + if (message is global::Google.Protobuf.IBufferMessage) + { + context.SetPayloadLength(message.CalculateSize()); + global::Google.Protobuf.MessageExtensions.WriteTo(message, context.GetBufferWriter()); + context.Complete(); + return; + } + #endif + context.Complete(global::Google.Protobuf.MessageExtensions.ToByteArray(message)); + } + + static class __Helper_MessageCache + { + public static readonly bool IsBufferMessage = global::System.Reflection.IntrospectionExtensions.GetTypeInfo(typeof(global::Google.Protobuf.IBufferMessage)).IsAssignableFrom(typeof(T)); + } + + static T __Helper_DeserializeMessage(grpc::DeserializationContext context, global::Google.Protobuf.MessageParser parser) where T : global::Google.Protobuf.IMessage + { + #if !GRPC_DISABLE_PROTOBUF_BUFFER_SERIALIZATION + if (__Helper_MessageCache.IsBufferMessage) + { + return parser.ParseFrom(context.PayloadAsReadOnlySequence()); + } + #endif + return parser.ParseFrom(context.PayloadAsNewBuffer()); + } + + static readonly grpc::Marshaller __Marshaller_grpc_reflection_v1alpha_ServerReflectionRequest = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::Grpc.Reflection.V1Alpha.ServerReflectionRequest.Parser)); + static readonly grpc::Marshaller __Marshaller_grpc_reflection_v1alpha_ServerReflectionResponse = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::Grpc.Reflection.V1Alpha.ServerReflectionResponse.Parser)); static readonly grpc::Method __Method_ServerReflectionInfo = new grpc::Method( grpc::MethodType.DuplexStreaming, From cd1b211321a3587702b5690742ea33012f332a8e Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Tue, 25 Aug 2020 12:08:09 -0700 Subject: [PATCH 30/62] Improve the exit test to yield better logs --- .../grpcio_tests/tests/unit/_exit_test.py | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/src/python/grpcio_tests/tests/unit/_exit_test.py b/src/python/grpcio_tests/tests/unit/_exit_test.py index da1658db812..9ba2cc65dfc 100644 --- a/src/python/grpcio_tests/tests/unit/_exit_test.py +++ b/src/python/grpcio_tests/tests/unit/_exit_test.py @@ -24,6 +24,7 @@ import six import subprocess import sys import threading +import datetime import time import unittest import logging @@ -37,7 +38,9 @@ INTERPRETER = sys.executable BASE_COMMAND = [INTERPRETER, SCENARIO_FILE] BASE_SIGTERM_COMMAND = BASE_COMMAND + ['--wait_for_interrupt'] -INIT_TIME = 1.0 +INIT_TIME = datetime.timedelta(seconds=1) +WAIT_CHECK_INTERVAL = datetime.timedelta(milliseconds=100) +WAIT_CHECK_DEFAULT_TIMEOUT = datetime.timedelta(seconds=5) processes = [] process_lock = threading.Lock() @@ -57,18 +60,27 @@ def cleanup_processes(): atexit.register(cleanup_processes) +def _process_wait_with_timeout(process, timeout=WAIT_CHECK_DEFAULT_TIMEOUT): + """A funciton to mimic 3.3+ only timeout argument in process.wait.""" + deadline = datetime.datetime.now() + timeout + while (process.poll() is None) and (datetime.datetime.now() < deadline): + time.sleep(WAIT_CHECK_INTERVAL.total_seconds()) + if process.returncode is None: + raise RuntimeError('Process failed to exit within %s' % timeout) + + def interrupt_and_wait(process): with process_lock: processes.append(process) - time.sleep(INIT_TIME) + time.sleep(INIT_TIME.total_seconds()) os.kill(process.pid, signal.SIGINT) - process.wait() + _process_wait_with_timeout(process) def wait(process): with process_lock: processes.append(process) - process.wait() + _process_wait_with_timeout(process) class ExitTest(unittest.TestCase): @@ -138,7 +150,6 @@ class ExitTest(unittest.TestCase): stderr=sys.stderr) interrupt_and_wait(process) - @unittest.skipIf(six.PY2, 'https://github.com/grpc/grpc/issues/6999') @unittest.skipIf(os.name == 'nt', 'os.kill does not have required permission on Windows') def test_in_flight_unary_stream_call(self): @@ -157,7 +168,6 @@ class ExitTest(unittest.TestCase): stderr=sys.stderr) interrupt_and_wait(process) - @unittest.skipIf(six.PY2, 'https://github.com/grpc/grpc/issues/6999') @unittest.skipIf(os.name == 'nt', 'os.kill does not have required permission on Windows') def test_in_flight_stream_stream_call(self): @@ -167,7 +177,6 @@ class ExitTest(unittest.TestCase): stderr=sys.stderr) interrupt_and_wait(process) - @unittest.skipIf(six.PY2, 'https://github.com/grpc/grpc/issues/6999') @unittest.skipIf(os.name == 'nt', 'os.kill does not have required permission on Windows') def test_in_flight_partial_unary_stream_call(self): @@ -188,7 +197,6 @@ class ExitTest(unittest.TestCase): stderr=sys.stderr) interrupt_and_wait(process) - @unittest.skipIf(six.PY2, 'https://github.com/grpc/grpc/issues/6999') @unittest.skipIf(os.name == 'nt', 'os.kill does not have required permission on Windows') def test_in_flight_partial_stream_stream_call(self): @@ -201,5 +209,5 @@ class ExitTest(unittest.TestCase): if __name__ == '__main__': - logging.basicConfig() + logging.basicConfig(level=logging.DEBUG) unittest.main(verbosity=2) From 1dbc68c4ae8dc31beebd0cecaec66d89649f5812 Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Tue, 25 Aug 2020 15:13:44 -0700 Subject: [PATCH 31/62] Disable two in flight stream stream cases due to known flake --- src/python/grpcio_tests/tests/unit/_exit_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/python/grpcio_tests/tests/unit/_exit_test.py b/src/python/grpcio_tests/tests/unit/_exit_test.py index 9ba2cc65dfc..146d30162ea 100644 --- a/src/python/grpcio_tests/tests/unit/_exit_test.py +++ b/src/python/grpcio_tests/tests/unit/_exit_test.py @@ -168,6 +168,7 @@ class ExitTest(unittest.TestCase): stderr=sys.stderr) interrupt_and_wait(process) + @unittest.skip('https://github.com/grpc/grpc/issues/23982') @unittest.skipIf(os.name == 'nt', 'os.kill does not have required permission on Windows') def test_in_flight_stream_stream_call(self): @@ -197,6 +198,7 @@ class ExitTest(unittest.TestCase): stderr=sys.stderr) interrupt_and_wait(process) + @unittest.skip('https://github.com/grpc/grpc/issues/23982') @unittest.skipIf(os.name == 'nt', 'os.kill does not have required permission on Windows') def test_in_flight_partial_stream_stream_call(self): From b83c481f2b418c1ad7b63149087205141358613b Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Tue, 25 Aug 2020 16:56:00 -0700 Subject: [PATCH 32/62] Remove stream from stalled lists on remove_stream --- src/core/ext/transport/chttp2/transport/chttp2_transport.cc | 2 ++ src/core/ext/transport/chttp2/transport/internal.h | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc index de96ca7b65a..068fcd76c6e 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc @@ -2030,6 +2030,8 @@ static void remove_stream(grpc_chttp2_transport* t, uint32_t id, if (grpc_chttp2_list_remove_writable_stream(t, s)) { GRPC_CHTTP2_STREAM_UNREF(s, "chttp2_writing:remove_stream"); } + grpc_chttp2_list_remove_stalled_by_stream(t, s); + grpc_chttp2_list_remove_stalled_by_transport(t, s); GRPC_ERROR_UNREF(error); diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index 28da24f647d..b04a88db5a7 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -52,8 +52,12 @@ class ContextList; /* streams are kept in various linked lists depending on what things need to happen to them... this enum labels each list */ typedef enum { + /* If a stream is in the following two lists, an explicit ref is associated + with the stream */ GRPC_CHTTP2_LIST_WRITABLE, GRPC_CHTTP2_LIST_WRITING, + /* No additional ref is taken for the following refs. Make sure to remove the + stream from these lists when the stream is removed. */ GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT, GRPC_CHTTP2_LIST_STALLED_BY_STREAM, /** streams that are waiting to start because there are too many concurrent From ca72e15e3ca7b116bdee1a465a5766fa2f4751c3 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Tue, 25 Aug 2020 17:09:22 -0700 Subject: [PATCH 33/62] Cleanup --- src/core/ext/transport/chttp2/transport/chttp2_transport.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc index 068fcd76c6e..570284aa07a 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc @@ -675,9 +675,6 @@ grpc_chttp2_stream::~grpc_chttp2_stream() { grpc_slice_buffer_destroy_internal(&decompressed_data_buffer); } - grpc_chttp2_list_remove_stalled_by_transport(t, this); - grpc_chttp2_list_remove_stalled_by_stream(t, this); - for (int i = 0; i < STREAM_LIST_COUNT; i++) { if (GPR_UNLIKELY(included[i])) { gpr_log(GPR_ERROR, "%s stream %d still included in list %d", From ecc3779b99af9b68682c0c7ad8724ac93e811137 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CMohan?= Date: Wed, 26 Aug 2020 03:01:44 -0700 Subject: [PATCH 34/62] remove linkstatic because its default is true --- bazel/grpc_build_system.bzl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl index 7ed433fedb7..ea13f3330c2 100644 --- a/bazel/grpc_build_system.bzl +++ b/bazel/grpc_build_system.bzl @@ -231,7 +231,7 @@ def grpc_cc_test(name, srcs = [], deps = [], external_deps = [], args = [], data **args ) -def grpc_cc_binary(name, srcs = [], deps = [], external_deps = [], args = [], data = [], language = "C++", testonly = False, linkshared = False, linkstatic = False, linkopts = [], tags = [], features = []): +def grpc_cc_binary(name, srcs = [], deps = [], external_deps = [], args = [], data = [], language = "C++", testonly = False, linkshared = False, linkopts = [], tags = [], features = []): copts = [] if language.upper() == "C": copts = ["-std=c99"] @@ -242,7 +242,6 @@ def grpc_cc_binary(name, srcs = [], deps = [], external_deps = [], args = [], da data = data, testonly = testonly, linkshared = linkshared, - linkstatic = linkstatic, deps = deps + _get_external_deps(external_deps), copts = copts, linkopts = if_not_windows(["-pthread"]) + linkopts, From 2fb9b57fe56a083a3641abe0801c8a887d6d3189 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 27 Aug 2020 11:05:08 +0200 Subject: [PATCH 35/62] upgrade Google.Protobuf in C# distribtests --- test/distrib/csharp/DistribTest/DistribTest.csproj | 2 +- test/distrib/csharp/DistribTest/DistribTestDotNet.csproj | 2 +- test/distrib/csharp/DistribTest/packages.config | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/distrib/csharp/DistribTest/DistribTest.csproj b/test/distrib/csharp/DistribTest/DistribTest.csproj index e65c1f8070c..41b45aa4581 100644 --- a/test/distrib/csharp/DistribTest/DistribTest.csproj +++ b/test/distrib/csharp/DistribTest/DistribTest.csproj @@ -101,7 +101,7 @@ ..\packages\Google.Apis.Auth.1.15.0\lib\net45\Google.Apis.Auth.PlatformServices.dll - ..\packages\Google.Protobuf.3.12.2\lib\net45\Google.Protobuf.dll + ..\packages\Google.Protobuf.3.13.0\lib\net45\Google.Protobuf.dll diff --git a/test/distrib/csharp/DistribTest/DistribTestDotNet.csproj b/test/distrib/csharp/DistribTest/DistribTestDotNet.csproj index 75c7973046f..5b14fff4444 100644 --- a/test/distrib/csharp/DistribTest/DistribTestDotNet.csproj +++ b/test/distrib/csharp/DistribTest/DistribTestDotNet.csproj @@ -16,7 +16,7 @@ - + diff --git a/test/distrib/csharp/DistribTest/packages.config b/test/distrib/csharp/DistribTest/packages.config index 07033764495..b0a9cd55c1b 100644 --- a/test/distrib/csharp/DistribTest/packages.config +++ b/test/distrib/csharp/DistribTest/packages.config @@ -8,7 +8,7 @@ - + From 42f741ecc413ca5a40fdcedd7fe9017e538851b5 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 27 Aug 2020 15:04:48 +0200 Subject: [PATCH 36/62] use GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE for C# distribtest that tests classic .csproj --- test/distrib/csharp/DistribTest/DistribTest.csproj | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/distrib/csharp/DistribTest/DistribTest.csproj b/test/distrib/csharp/DistribTest/DistribTest.csproj index 41b45aa4581..e083d6225e7 100644 --- a/test/distrib/csharp/DistribTest/DistribTest.csproj +++ b/test/distrib/csharp/DistribTest/DistribTest.csproj @@ -56,6 +56,12 @@ true 4 + + + $(DefineConstants);GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + ..\packages\Grpc.Auth.__GRPC_NUGET_VERSION__\lib\net45\Grpc.Auth.dll From 369bd24b7cafcaefc73a5cff758e15099e965c79 Mon Sep 17 00:00:00 2001 From: Ashitha Santhosh Date: Wed, 26 Aug 2020 17:11:40 -0700 Subject: [PATCH 37/62] Add mock CEL APIs. --- BUILD | 21 +++++- BUILD.gn | 5 ++ build_autogenerated.yaml | 5 ++ gRPC-C++.podspec | 10 +++ gRPC-Core.podspec | 10 +++ grpc.gemspec | 5 ++ package.xml | 5 ++ .../mock_cel/cel_expr_builder_factory.h | 42 ++++++++++++ .../authorization/mock_cel/cel_expression.h | 68 +++++++++++++++++++ .../authorization/mock_cel/evaluator_core.h | 67 ++++++++++++++++++ .../mock_cel/flat_expr_builder.h | 56 +++++++++++++++ .../authorization/mock_cel/statusor.h | 50 ++++++++++++++ tools/doxygen/Doxyfile.c++.internal | 5 ++ tools/doxygen/Doxyfile.core.internal | 5 ++ 14 files changed, 352 insertions(+), 2 deletions(-) create mode 100644 src/core/lib/security/authorization/mock_cel/cel_expr_builder_factory.h create mode 100644 src/core/lib/security/authorization/mock_cel/cel_expression.h create mode 100644 src/core/lib/security/authorization/mock_cel/evaluator_core.h create mode 100644 src/core/lib/security/authorization/mock_cel/flat_expr_builder.h create mode 100644 src/core/lib/security/authorization/mock_cel/statusor.h diff --git a/BUILD b/BUILD index 5ec511c504e..384061e14a2 100644 --- a/BUILD +++ b/BUILD @@ -1853,6 +1853,24 @@ grpc_cc_library( ], ) +grpc_cc_library( + name = "grpc_mock_cel", + hdrs = [ + "src/core/lib/security/authorization/mock_cel/activation.h", + "src/core/lib/security/authorization/mock_cel/cel_expr_builder_factory.h", + "src/core/lib/security/authorization/mock_cel/cel_expression.h", + "src/core/lib/security/authorization/mock_cel/cel_value.h", + "src/core/lib/security/authorization/mock_cel/evaluator_core.h", + "src/core/lib/security/authorization/mock_cel/flat_expr_builder.h", + "src/core/lib/security/authorization/mock_cel/statusor.h", + ], + language = "c++", + deps = [ + "google_api_upb", + "grpc_base", + ], +) + grpc_cc_library( name = "grpc_authorization_engine", srcs = [ @@ -1862,8 +1880,6 @@ grpc_cc_library( hdrs = [ "src/core/lib/security/authorization/authorization_engine.h", "src/core/lib/security/authorization/evaluate_args.h", - "src/core/lib/security/authorization/mock_cel/activation.h", - "src/core/lib/security/authorization/mock_cel/cel_value.h", ], external_deps = [ "absl/container:flat_hash_set", @@ -1873,6 +1889,7 @@ grpc_cc_library( "envoy_ads_upb", "google_api_upb", "grpc_base", + "grpc_mock_cel", "grpc_secure", ], ) diff --git a/BUILD.gn b/BUILD.gn index 56d77fc6700..7992a11f1ee 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -793,7 +793,12 @@ config("grpc_config") { "src/core/lib/security/authorization/evaluate_args.cc", "src/core/lib/security/authorization/evaluate_args.h", "src/core/lib/security/authorization/mock_cel/activation.h", + "src/core/lib/security/authorization/mock_cel/cel_expr_builder_factory.h", + "src/core/lib/security/authorization/mock_cel/cel_expression.h", "src/core/lib/security/authorization/mock_cel/cel_value.h", + "src/core/lib/security/authorization/mock_cel/evaluator_core.h", + "src/core/lib/security/authorization/mock_cel/flat_expr_builder.h", + "src/core/lib/security/authorization/mock_cel/statusor.h", "src/core/lib/security/context/security_context.cc", "src/core/lib/security/context/security_context.h", "src/core/lib/security/credentials/alts/alts_credentials.cc", diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index f62b8017c15..aea2c9bcd51 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -654,7 +654,12 @@ libs: - src/core/lib/security/authorization/authorization_engine.h - src/core/lib/security/authorization/evaluate_args.h - src/core/lib/security/authorization/mock_cel/activation.h + - src/core/lib/security/authorization/mock_cel/cel_expr_builder_factory.h + - src/core/lib/security/authorization/mock_cel/cel_expression.h - src/core/lib/security/authorization/mock_cel/cel_value.h + - src/core/lib/security/authorization/mock_cel/evaluator_core.h + - src/core/lib/security/authorization/mock_cel/flat_expr_builder.h + - src/core/lib/security/authorization/mock_cel/statusor.h - src/core/lib/security/context/security_context.h - src/core/lib/security/credentials/alts/alts_credentials.h - src/core/lib/security/credentials/alts/check_gcp_environment.h diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index 37efd951d34..42bb53c9fe3 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -517,7 +517,12 @@ Pod::Spec.new do |s| 'src/core/lib/security/authorization/authorization_engine.h', 'src/core/lib/security/authorization/evaluate_args.h', 'src/core/lib/security/authorization/mock_cel/activation.h', + 'src/core/lib/security/authorization/mock_cel/cel_expr_builder_factory.h', + 'src/core/lib/security/authorization/mock_cel/cel_expression.h', 'src/core/lib/security/authorization/mock_cel/cel_value.h', + 'src/core/lib/security/authorization/mock_cel/evaluator_core.h', + 'src/core/lib/security/authorization/mock_cel/flat_expr_builder.h', + 'src/core/lib/security/authorization/mock_cel/statusor.h', 'src/core/lib/security/context/security_context.h', 'src/core/lib/security/credentials/alts/alts_credentials.h', 'src/core/lib/security/credentials/alts/check_gcp_environment.h', @@ -1011,7 +1016,12 @@ Pod::Spec.new do |s| 'src/core/lib/security/authorization/authorization_engine.h', 'src/core/lib/security/authorization/evaluate_args.h', 'src/core/lib/security/authorization/mock_cel/activation.h', + 'src/core/lib/security/authorization/mock_cel/cel_expr_builder_factory.h', + 'src/core/lib/security/authorization/mock_cel/cel_expression.h', 'src/core/lib/security/authorization/mock_cel/cel_value.h', + 'src/core/lib/security/authorization/mock_cel/evaluator_core.h', + 'src/core/lib/security/authorization/mock_cel/flat_expr_builder.h', + 'src/core/lib/security/authorization/mock_cel/statusor.h', 'src/core/lib/security/context/security_context.h', 'src/core/lib/security/credentials/alts/alts_credentials.h', 'src/core/lib/security/credentials/alts/check_gcp_environment.h', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index bd418e8f858..558884b7226 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -848,7 +848,12 @@ Pod::Spec.new do |s| 'src/core/lib/security/authorization/evaluate_args.cc', 'src/core/lib/security/authorization/evaluate_args.h', 'src/core/lib/security/authorization/mock_cel/activation.h', + 'src/core/lib/security/authorization/mock_cel/cel_expr_builder_factory.h', + 'src/core/lib/security/authorization/mock_cel/cel_expression.h', 'src/core/lib/security/authorization/mock_cel/cel_value.h', + 'src/core/lib/security/authorization/mock_cel/evaluator_core.h', + 'src/core/lib/security/authorization/mock_cel/flat_expr_builder.h', + 'src/core/lib/security/authorization/mock_cel/statusor.h', 'src/core/lib/security/context/security_context.cc', 'src/core/lib/security/context/security_context.h', 'src/core/lib/security/credentials/alts/alts_credentials.cc', @@ -1422,7 +1427,12 @@ Pod::Spec.new do |s| 'src/core/lib/security/authorization/authorization_engine.h', 'src/core/lib/security/authorization/evaluate_args.h', 'src/core/lib/security/authorization/mock_cel/activation.h', + 'src/core/lib/security/authorization/mock_cel/cel_expr_builder_factory.h', + 'src/core/lib/security/authorization/mock_cel/cel_expression.h', 'src/core/lib/security/authorization/mock_cel/cel_value.h', + 'src/core/lib/security/authorization/mock_cel/evaluator_core.h', + 'src/core/lib/security/authorization/mock_cel/flat_expr_builder.h', + 'src/core/lib/security/authorization/mock_cel/statusor.h', 'src/core/lib/security/context/security_context.h', 'src/core/lib/security/credentials/alts/alts_credentials.h', 'src/core/lib/security/credentials/alts/check_gcp_environment.h', diff --git a/grpc.gemspec b/grpc.gemspec index a97ae76320f..3fe56c115fb 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -766,7 +766,12 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/security/authorization/evaluate_args.cc ) s.files += %w( src/core/lib/security/authorization/evaluate_args.h ) s.files += %w( src/core/lib/security/authorization/mock_cel/activation.h ) + s.files += %w( src/core/lib/security/authorization/mock_cel/cel_expr_builder_factory.h ) + s.files += %w( src/core/lib/security/authorization/mock_cel/cel_expression.h ) s.files += %w( src/core/lib/security/authorization/mock_cel/cel_value.h ) + s.files += %w( src/core/lib/security/authorization/mock_cel/evaluator_core.h ) + s.files += %w( src/core/lib/security/authorization/mock_cel/flat_expr_builder.h ) + s.files += %w( src/core/lib/security/authorization/mock_cel/statusor.h ) s.files += %w( src/core/lib/security/context/security_context.cc ) s.files += %w( src/core/lib/security/context/security_context.h ) s.files += %w( src/core/lib/security/credentials/alts/alts_credentials.cc ) diff --git a/package.xml b/package.xml index ed80ad00315..5c3d26ca57c 100644 --- a/package.xml +++ b/package.xml @@ -746,7 +746,12 @@ + + + + + diff --git a/src/core/lib/security/authorization/mock_cel/cel_expr_builder_factory.h b/src/core/lib/security/authorization/mock_cel/cel_expr_builder_factory.h new file mode 100644 index 00000000000..d9d99726787 --- /dev/null +++ b/src/core/lib/security/authorization/mock_cel/cel_expr_builder_factory.h @@ -0,0 +1,42 @@ +// Copyright 2020 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GRPC_CORE_LIB_SECURITY_AUTHORIZATION_MOCK_CEL_CEL_EXPR_BUILDER_FACTORY_H +#define GRPC_CORE_LIB_SECURITY_AUTHORIZATION_MOCK_CEL_CEL_EXPR_BUILDER_FACTORY_H + +#include + +#include + +#include "src/core/lib/security/authorization/mock_cel/flat_expr_builder.h" + +namespace grpc_core { +namespace mock_cel { + +// This is a temporary stub implementation of CEL APIs. +// Once gRPC imports the CEL library, this file will be removed. + +struct InterpreterOptions { + bool short_circuiting = true; +}; + +std::unique_ptr CreateCelExpressionBuilder( + const InterpreterOptions& options) { + return absl::make_unique(); +} + +} // namespace mock_cel +} // namespace grpc_core + +#endif // GRPC_CORE_LIB_SECURITY_AUTHORIZATION_MOCK_CEL_CEL_EXPR_BUILDER_FACTORY_H \ No newline at end of file diff --git a/src/core/lib/security/authorization/mock_cel/cel_expression.h b/src/core/lib/security/authorization/mock_cel/cel_expression.h new file mode 100644 index 00000000000..bdeba67a41d --- /dev/null +++ b/src/core/lib/security/authorization/mock_cel/cel_expression.h @@ -0,0 +1,68 @@ +// Copyright 2020 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GRPC_CORE_LIB_SECURITY_AUTHORIZATION_MOCK_CEL_CEL_EXPRESSION_H +#define GRPC_CORE_LIB_SECURITY_AUTHORIZATION_MOCK_CEL_CEL_EXPRESSION_H + +#include + +#include +#include + +#include "google/api/expr/v1alpha1/syntax.upb.h" +#include "src/core/lib/security/authorization/mock_cel/activation.h" +#include "src/core/lib/security/authorization/mock_cel/cel_value.h" +#include "src/core/lib/security/authorization/mock_cel/statusor.h" + +namespace grpc_core { +namespace mock_cel { + +// This is a temporary stub implementation of CEL APIs. +// Once gRPC imports the CEL library, this file will be removed. + +// Base interface for expression evaluating objects. +class CelExpression { + public: + virtual ~CelExpression() = default; + + // Evaluates expression and returns value. + // activation contains bindings from parameter names to values + virtual StatusOr Evaluate( + const BaseActivation& activation) const = 0; +}; + +// Base class for Expression Builder implementations +// Provides user with factory to register extension functions. +// ExpressionBuilder MUST NOT be destroyed before CelExpression objects +// it built. +class CelExpressionBuilder { + public: + virtual ~CelExpressionBuilder() = default; + + // Creates CelExpression object from AST tree. + // expr specifies root of AST tree + virtual StatusOr> CreateExpression( + const google_api_expr_v1alpha1_Expr* expr, + const google_api_expr_v1alpha1_SourceInfo* source_info) const = 0; + + virtual StatusOr> CreateExpression( + const google_api_expr_v1alpha1_Expr* expr, + const google_api_expr_v1alpha1_SourceInfo* source_info, + std::vector* warnings) const = 0; +}; + +} // namespace mock_cel +} // namespace grpc_core + +#endif // GRPC_CORE_LIB_SECURITY_AUTHORIZATION_MOCK_CEL_CEL_EXPRESSION_H \ No newline at end of file diff --git a/src/core/lib/security/authorization/mock_cel/evaluator_core.h b/src/core/lib/security/authorization/mock_cel/evaluator_core.h new file mode 100644 index 00000000000..7ddf495baf0 --- /dev/null +++ b/src/core/lib/security/authorization/mock_cel/evaluator_core.h @@ -0,0 +1,67 @@ +// Copyright 2020 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GRPC_CORE_LIB_SECURITY_AUTHORIZATION_MOCK_CEL_EVALUATOR_CORE_H +#define GRPC_CORE_LIB_SECURITY_AUTHORIZATION_MOCK_CEL_EVALUATOR_CORE_H + +#include + +#include +#include +#include + +#include "google/api/expr/v1alpha1/syntax.upb.h" +#include "src/core/lib/security/authorization/mock_cel/activation.h" +#include "src/core/lib/security/authorization/mock_cel/cel_expression.h" +#include "src/core/lib/security/authorization/mock_cel/cel_value.h" +#include "src/core/lib/security/authorization/mock_cel/statusor.h" + +namespace grpc_core { +namespace mock_cel { + +// This is a temporary stub implementation of CEL APIs. +// Once gRPC imports the CEL library, this file will be removed. + +class ExpressionStep { + public: + virtual ~ExpressionStep() = default; +}; + +using ExecutionPath = std::vector>; + +// Implementation of the CelExpression that utilizes flattening +// of the expression tree. +class CelExpressionFlatImpl : public CelExpression { + // Constructs CelExpressionFlatImpl instance. + // path is flat execution path that is based upon + // flattened AST tree. Max iterations dictates the maximum number of + // iterations in the comprehension expressions (use 0 to disable the upper + // bound). + public: + CelExpressionFlatImpl(const google_api_expr_v1alpha1_Expr* root_expr, + ExecutionPath path, int max_iterations, + std::set iter_variable_names, + bool enable_unknowns = false, + bool enable_unknown_function_results = false) {} + + // Implementation of CelExpression evaluate method. + StatusOr Evaluate(const BaseActivation& activation) const override { + return CelValue::CreateNull(); + } +}; + +} // namespace mock_cel +} // namespace grpc_core + +#endif // GRPC_CORE_LIB_SECURITY_AUTHORIZATION_MOCK_CEL_EVALUATOR_CORE_H \ No newline at end of file diff --git a/src/core/lib/security/authorization/mock_cel/flat_expr_builder.h b/src/core/lib/security/authorization/mock_cel/flat_expr_builder.h new file mode 100644 index 00000000000..86f5a239a32 --- /dev/null +++ b/src/core/lib/security/authorization/mock_cel/flat_expr_builder.h @@ -0,0 +1,56 @@ +// Copyright 2020 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GRPC_CORE_LIB_SECURITY_AUTHORIZATION_MOCK_CEL_FLAT_EXPR_BUILDER_H +#define GRPC_CORE_LIB_SECURITY_AUTHORIZATION_MOCK_CEL_FLAT_EXPR_BUILDER_H + +#include + +#include + +#include "src/core/lib/security/authorization/mock_cel/evaluator_core.h" + +namespace grpc_core { +namespace mock_cel { + +// This is a temporary stub implementation of CEL APIs. +// Once gRPC imports the CEL library, this file will be removed. + +// CelExpressionBuilder implementation. +// Builds instances of CelExpressionFlatImpl. +class FlatExprBuilder : public CelExpressionBuilder { + public: + FlatExprBuilder() = default; + + cel_base::StatusOr> CreateExpression( + const google::api::expr::v1alpha1::Expr* expr, + const google::api::expr::v1alpha1::SourceInfo* source_info) + const override { + ExecutionPath path; + return absl::make_unique(nullptr, path, 0); + } + + cel_base::StatusOr> CreateExpression( + const google::api::expr::v1alpha1::Expr* expr, + const google::api::expr::v1alpha1::SourceInfo* source_info, + std::vector* warnings) const override { + ExecutionPath path; + return absl::make_unique(nullptr, path, 0); + } +}; + +} // namespace mock_cel +} // namespace grpc_core + +#endif // GRPC_CORE_LIB_SECURITY_AUTHORIZATION_MOCK_CEL_FLAT_EXPR_BUILDER_H diff --git a/src/core/lib/security/authorization/mock_cel/statusor.h b/src/core/lib/security/authorization/mock_cel/statusor.h new file mode 100644 index 00000000000..f97a8ec1229 --- /dev/null +++ b/src/core/lib/security/authorization/mock_cel/statusor.h @@ -0,0 +1,50 @@ +// Copyright 2020 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GRPC_CORE_LIB_SECURITY_AUTHORIZATION_MOCK_CEL_STATUSOR_H +#define GRPC_CORE_LIB_SECURITY_AUTHORIZATION_MOCK_CEL_STATUSOR_H + +#include + +#include + +#include "absl/status/status.h" + +namespace grpc_core { +namespace mock_cel { + +// This is a temporary stub implementation of CEL APIs. +// Once gRPC imports the CEL library, this file will be removed. + +template +class ABSL_MUST_USE_RESULT StatusOr; + +template +class StatusOr { + public: + StatusOr() = default; + + StatusOr(const T& value) {} + + StatusOr(const absl::Status& status) {} + + StatusOr(absl::Status&& status) {} + + bool ok() const { return true; } +}; + +} // namespace mock_cel +} // namespace grpc_core + +#endif // GRPC_CORE_LIB_SECURITY_AUTHORIZATION_MOCK_CEL_STATUSOR_H \ No newline at end of file diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index b645161be23..5876739a4b8 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -1715,7 +1715,12 @@ src/core/lib/security/authorization/authorization_engine.h \ src/core/lib/security/authorization/evaluate_args.cc \ src/core/lib/security/authorization/evaluate_args.h \ src/core/lib/security/authorization/mock_cel/activation.h \ +src/core/lib/security/authorization/mock_cel/cel_expr_builder_factory.h \ +src/core/lib/security/authorization/mock_cel/cel_expression.h \ src/core/lib/security/authorization/mock_cel/cel_value.h \ +src/core/lib/security/authorization/mock_cel/evaluator_core.h \ +src/core/lib/security/authorization/mock_cel/flat_expr_builder.h \ +src/core/lib/security/authorization/mock_cel/statusor.h \ src/core/lib/security/context/security_context.cc \ src/core/lib/security/context/security_context.h \ src/core/lib/security/credentials/alts/alts_credentials.cc \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index f4afb777a81..c663f08e41f 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1542,7 +1542,12 @@ src/core/lib/security/authorization/authorization_engine.h \ src/core/lib/security/authorization/evaluate_args.cc \ src/core/lib/security/authorization/evaluate_args.h \ src/core/lib/security/authorization/mock_cel/activation.h \ +src/core/lib/security/authorization/mock_cel/cel_expr_builder_factory.h \ +src/core/lib/security/authorization/mock_cel/cel_expression.h \ src/core/lib/security/authorization/mock_cel/cel_value.h \ +src/core/lib/security/authorization/mock_cel/evaluator_core.h \ +src/core/lib/security/authorization/mock_cel/flat_expr_builder.h \ +src/core/lib/security/authorization/mock_cel/statusor.h \ src/core/lib/security/context/security_context.cc \ src/core/lib/security/context/security_context.h \ src/core/lib/security/credentials/alts/alts_credentials.cc \ From f8e684944c903500a83b305b25ba753d45acd3f4 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Tue, 25 Aug 2020 11:02:15 -0700 Subject: [PATCH 38/62] Upgraded gcc to 8 in python_dev_centos7_x64 --- .../python_dev_centos7_x64/Dockerfile | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/tools/dockerfile/distribtest/python_dev_centos7_x64/Dockerfile b/tools/dockerfile/distribtest/python_dev_centos7_x64/Dockerfile index 954146c5e91..18e3c7b05c6 100644 --- a/tools/dockerfile/distribtest/python_dev_centos7_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_dev_centos7_x64/Dockerfile @@ -15,8 +15,25 @@ FROM centos:7 RUN yum install -y python +RUN yum install -y python-devel RUN yum install -y epel-release RUN yum install -y python-pip -RUN pip install virtualenv -RUN yum groupinstall -y 'Development Tools' -RUN yum install -y python-devel +RUN pip install --upgrade pip +RUN pip install -U virtualenv + +# The default gcc of CentOS 7 is gcc 4.8 which is older than gcc 4.9, +# the minimum supported gcc version for gRPC Core so let's upgrade to +# the oldest one that can build gRPC on Centos 7. +RUN yum install -y centos-release-scl +RUN yum install -y devtoolset-8-binutils devtoolset-8-gcc devtoolset-8-gcc-c++ + +# Activate devtoolset-8 by default +# https://austindewey.com/2019/03/26/enabling-software-collections-binaries-on-a-docker-image/ +RUN echo $'#!/bin/bash\n\ +source scl_source enable devtoolset-8\n\ +"$@"\n' > /usr/bin/entrypoint.sh +RUN chmod +x /usr/bin/entrypoint.sh +RUN cat /usr/bin/entrypoint.sh +ENTRYPOINT [ "/usr/bin/entrypoint.sh" ] + +CMD ["/bin/bash"] From 30396fb025d6e7f2a9dd155ef3b0e6973ebc586b Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Thu, 27 Aug 2020 10:47:44 -0700 Subject: [PATCH 39/62] Update grpc_android for new CI environment --- tools/internal_ci/linux/grpc_android.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/internal_ci/linux/grpc_android.sh b/tools/internal_ci/linux/grpc_android.sh index 209b30d1ad7..d335e783947 100755 --- a/tools/internal_ci/linux/grpc_android.sh +++ b/tools/internal_ci/linux/grpc_android.sh @@ -15,6 +15,10 @@ set -ex +# Install packages which werer not preinstalled yet. +# Protobuf needs autoconf & automake to build +sudo apt-get install -y autoconf automake + # change to grpc repo root cd $(dirname $0)/../../.. From b1fd4a45a45cbba56cea45bc6b731d30eae9c8cc Mon Sep 17 00:00:00 2001 From: Ashitha Santhosh Date: Tue, 25 Aug 2020 01:39:08 -0700 Subject: [PATCH 40/62] Add EvaluateArgs test. --- test/core/security/evaluate_args_test.cc | 165 +++++++++++++++++++++-- 1 file changed, 156 insertions(+), 9 deletions(-) diff --git a/test/core/security/evaluate_args_test.cc b/test/core/security/evaluate_args_test.cc index 394344dd60e..ff696c390af 100644 --- a/test/core/security/evaluate_args_test.cc +++ b/test/core/security/evaluate_args_test.cc @@ -14,7 +14,9 @@ #include +#include #include + #include "absl/strings/string_view.h" #include "src/core/lib/security/authorization/evaluate_args.h" @@ -45,27 +47,172 @@ class EvaluateArgsTest : public ::testing::Test { TEST_F(EvaluateArgsTest, TestEvaluateArgsLocalAddress) { absl::string_view src_address = evaluate_args_->GetLocalAddress(); - EXPECT_EQ(src_address, local_address_) - << "Error: Failed to extract correct Local address from EvaluateArgs."; + EXPECT_EQ(src_address, local_address_); } TEST_F(EvaluateArgsTest, TestEvaluateArgsLocalPort) { int src_port = evaluate_args_->GetLocalPort(); - EXPECT_EQ(src_port, local_port_) - << "Error: Failed to extract correct Local port from EvaluateArgs."; + EXPECT_EQ(src_port, local_port_); } TEST_F(EvaluateArgsTest, TestEvaluateArgsPeerAddress) { absl::string_view dest_address = evaluate_args_->GetPeerAddress(); - EXPECT_EQ(dest_address, peer_address_) - << "Error: Failed to extract correct Peer address from " - "EvaluateArgs. "; + EXPECT_EQ(dest_address, peer_address_); } TEST_F(EvaluateArgsTest, TestEvaluateArgsPeerPort) { int dest_port = evaluate_args_->GetPeerPort(); - EXPECT_EQ(dest_port, peer_port_) - << "Error: Failed to extract correct Peer port from EvaluateArgs."; + EXPECT_EQ(dest_port, peer_port_); +} + +TEST(EvaluateArgsMetadataTest, HandlesNullMetadata) { + EvaluateArgs eval_args(nullptr, nullptr, nullptr); + EXPECT_EQ(eval_args.GetPath(), nullptr); + EXPECT_EQ(eval_args.GetMethod(), nullptr); + EXPECT_EQ(eval_args.GetHost(), nullptr); + EXPECT_THAT(eval_args.GetHeaders(), ::testing::ElementsAre()); +} + +TEST(EvaluateArgsMetadataTest, HandlesEmptyMetadata) { + grpc_metadata_batch metadata; + grpc_metadata_batch_init(&metadata); + EvaluateArgs eval_args(&metadata, nullptr, nullptr); + EXPECT_EQ(eval_args.GetPath(), nullptr); + EXPECT_EQ(eval_args.GetMethod(), nullptr); + EXPECT_EQ(eval_args.GetHost(), nullptr); + EXPECT_THAT(eval_args.GetHeaders(), ::testing::ElementsAre()); + grpc_metadata_batch_destroy(&metadata); +} + +TEST(EvaluateArgsMetadataTest, GetPathSuccess) { + grpc_init(); + const char* kPath = "/some/path"; + grpc_metadata_batch metadata; + grpc_metadata_batch_init(&metadata); + grpc_slice fake_val = grpc_slice_intern(grpc_slice_from_static_string(kPath)); + grpc_mdelem fake_val_md = grpc_mdelem_from_slices(GRPC_MDSTR_PATH, fake_val); + grpc_linked_mdelem storage; + storage.md = fake_val_md; + ASSERT_EQ(grpc_metadata_batch_link_head(&metadata, &storage), + GRPC_ERROR_NONE); + EvaluateArgs eval_args(&metadata, nullptr, nullptr); + EXPECT_EQ(eval_args.GetPath(), kPath); + grpc_metadata_batch_destroy(&metadata); + grpc_shutdown(); +} + +TEST(EvaluateArgsMetadataTest, GetHostSuccess) { + grpc_init(); + const char* kHost = "host"; + grpc_metadata_batch metadata; + grpc_metadata_batch_init(&metadata); + grpc_slice fake_val = grpc_slice_intern(grpc_slice_from_static_string(kHost)); + grpc_mdelem fake_val_md = grpc_mdelem_from_slices(GRPC_MDSTR_HOST, fake_val); + grpc_linked_mdelem storage; + storage.md = fake_val_md; + ASSERT_EQ(grpc_metadata_batch_link_head(&metadata, &storage), + GRPC_ERROR_NONE); + EvaluateArgs eval_args(&metadata, nullptr, nullptr); + EXPECT_EQ(eval_args.GetHost(), kHost); + grpc_metadata_batch_destroy(&metadata); + grpc_shutdown(); +} + +TEST(EvaluateArgsMetadataTest, GetMethodSuccess) { + grpc_init(); + const char* kMethod = "GET"; + grpc_metadata_batch metadata; + grpc_metadata_batch_init(&metadata); + grpc_slice fake_val = + grpc_slice_intern(grpc_slice_from_static_string(kMethod)); + grpc_mdelem fake_val_md = + grpc_mdelem_from_slices(GRPC_MDSTR_METHOD, fake_val); + grpc_linked_mdelem storage; + storage.md = fake_val_md; + ASSERT_EQ(grpc_metadata_batch_link_head(&metadata, &storage), + GRPC_ERROR_NONE); + EvaluateArgs eval_args(&metadata, nullptr, nullptr); + EXPECT_EQ(eval_args.GetMethod(), kMethod); + grpc_metadata_batch_destroy(&metadata); + grpc_shutdown(); +} + +TEST(EvaluateArgsMetadataTest, GetHeadersSuccess) { + grpc_init(); + const char* kPath = "/some/path"; + const char* kHost = "host"; + grpc_metadata_batch metadata; + grpc_metadata_batch_init(&metadata); + grpc_slice fake_path = + grpc_slice_intern(grpc_slice_from_static_string(kPath)); + grpc_mdelem fake_path_md = + grpc_mdelem_from_slices(GRPC_MDSTR_PATH, fake_path); + grpc_linked_mdelem storage; + storage.md = fake_path_md; + ASSERT_EQ(grpc_metadata_batch_link_head(&metadata, &storage, GRPC_BATCH_PATH), + GRPC_ERROR_NONE); + grpc_slice fake_host = + grpc_slice_intern(grpc_slice_from_static_string(kHost)); + grpc_mdelem fake_host_md = + grpc_mdelem_from_slices(GRPC_MDSTR_HOST, fake_host); + grpc_linked_mdelem storage2; + storage2.md = fake_host_md; + ASSERT_EQ( + grpc_metadata_batch_link_tail(&metadata, &storage2, GRPC_BATCH_HOST), + GRPC_ERROR_NONE); + EvaluateArgs eval_args(&metadata, nullptr, nullptr); + EXPECT_THAT( + eval_args.GetHeaders(), + ::testing::UnorderedElementsAre( + ::testing::Pair(StringViewFromSlice(GRPC_MDSTR_HOST), kHost), + ::testing::Pair(StringViewFromSlice(GRPC_MDSTR_PATH), kPath))); + grpc_metadata_batch_destroy(&metadata); + grpc_shutdown(); +} + +TEST(EvaluateArgsAuthContextTest, HandlesNullAuthContext) { + EvaluateArgs eval_args(nullptr, nullptr, nullptr); + EXPECT_EQ(eval_args.GetSpiffeId(), nullptr); + EXPECT_EQ(eval_args.GetCertServerName(), nullptr); +} + +TEST(EvaluateArgsAuthContextTest, HandlesEmptyAuthCtx) { + grpc_auth_context auth_context(nullptr); + EvaluateArgs eval_args(nullptr, &auth_context, nullptr); + EXPECT_EQ(eval_args.GetSpiffeId(), nullptr); + EXPECT_EQ(eval_args.GetCertServerName(), nullptr); +} + +TEST(EvaluateArgsAuthContextTest, GetSpiffeIdSuccessOneProperty) { + grpc_auth_context auth_context(nullptr); + const char* kId = "spiffeid"; + auth_context.add_cstring_property(GRPC_PEER_SPIFFE_ID_PROPERTY_NAME, kId); + EvaluateArgs eval_args(nullptr, &auth_context, nullptr); + EXPECT_EQ(eval_args.GetSpiffeId(), kId); +} + +TEST(EvaluateArgsAuthContextTest, GetSpiffeIdFailDuplicateProperty) { + grpc_auth_context auth_context(nullptr); + auth_context.add_cstring_property(GRPC_PEER_SPIFFE_ID_PROPERTY_NAME, "id1"); + auth_context.add_cstring_property(GRPC_PEER_SPIFFE_ID_PROPERTY_NAME, "id2"); + EvaluateArgs eval_args(nullptr, &auth_context, nullptr); + EXPECT_EQ(eval_args.GetSpiffeId(), nullptr); +} + +TEST(EvaluateArgsAuthContextTest, GetCertServerNameSuccessOneProperty) { + grpc_auth_context auth_context(nullptr); + const char* kServer = "server"; + auth_context.add_cstring_property(GRPC_X509_CN_PROPERTY_NAME, kServer); + EvaluateArgs eval_args(nullptr, &auth_context, nullptr); + EXPECT_EQ(eval_args.GetCertServerName(), kServer); +} + +TEST(EvaluateArgsAuthContextTest, GetCertServerNameFailDuplicateProperty) { + grpc_auth_context auth_context(nullptr); + auth_context.add_cstring_property(GRPC_X509_CN_PROPERTY_NAME, "server1"); + auth_context.add_cstring_property(GRPC_X509_CN_PROPERTY_NAME, "server2"); + EvaluateArgs eval_args(nullptr, &auth_context, nullptr); + EXPECT_EQ(eval_args.GetCertServerName(), nullptr); } } // namespace grpc_core From 51531aada02ca214c7e9e1a209c8689f3690e80f Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Thu, 27 Aug 2020 13:40:57 -0700 Subject: [PATCH 41/62] Upgrade gradle to 3.1.4 --- src/android/test/interop/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/android/test/interop/build.gradle b/src/android/test/interop/build.gradle index bd5f337f013..78591e3b824 100644 --- a/src/android/test/interop/build.gradle +++ b/src/android/test/interop/build.gradle @@ -6,7 +6,7 @@ buildscript { jcenter() } dependencies { - classpath 'com.android.tools.build:gradle:3.0.1' + classpath 'com.android.tools.build:gradle:3.1.4' // NOTE: Do not place your application dependencies here; they belong // in the individual module build.gradle files } From 10d7e2fef7118aff176bf99432875fc6ae6fe360 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Thu, 27 Aug 2020 14:15:44 -0700 Subject: [PATCH 42/62] Upgrade gradlewrapper to 6.5.1 --- .../android/helloworld/gradle/wrapper/gradle-wrapper.properties | 2 +- .../test/interop/gradle/wrapper/gradle-wrapper.properties | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/android/helloworld/gradle/wrapper/gradle-wrapper.properties b/examples/android/helloworld/gradle/wrapper/gradle-wrapper.properties index 4c81831f533..ac3220ae044 100644 --- a/examples/android/helloworld/gradle/wrapper/gradle-wrapper.properties +++ b/examples/android/helloworld/gradle/wrapper/gradle-wrapper.properties @@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-4.1-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-6.5.1-bin.zip diff --git a/src/android/test/interop/gradle/wrapper/gradle-wrapper.properties b/src/android/test/interop/gradle/wrapper/gradle-wrapper.properties index 4c81831f533..ac3220ae044 100644 --- a/src/android/test/interop/gradle/wrapper/gradle-wrapper.properties +++ b/src/android/test/interop/gradle/wrapper/gradle-wrapper.properties @@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-4.1-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-6.5.1-bin.zip From 3fbaffa24b9793f8e9d98b86fa8b967f88957b80 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Thu, 27 Aug 2020 14:16:28 -0700 Subject: [PATCH 43/62] Update by review --- tools/internal_ci/linux/grpc_android.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/internal_ci/linux/grpc_android.sh b/tools/internal_ci/linux/grpc_android.sh index d335e783947..81acbed6551 100755 --- a/tools/internal_ci/linux/grpc_android.sh +++ b/tools/internal_ci/linux/grpc_android.sh @@ -15,7 +15,7 @@ set -ex -# Install packages which werer not preinstalled yet. +# Install packages which were not preinstalled yet. # Protobuf needs autoconf & automake to build sudo apt-get install -y autoconf automake From 768e62e9ec0b287f1917af1a57b1c9d9aa9033ce Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Thu, 27 Aug 2020 14:47:12 -0700 Subject: [PATCH 44/62] Fix keepalive value used in XdsClient. --- src/core/ext/xds/xds_client.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/ext/xds/xds_client.cc b/src/core/ext/xds/xds_client.cc index 061b4c06005..e5c0c70a71b 100644 --- a/src/core/ext/xds/xds_client.cc +++ b/src/core/ext/xds/xds_client.cc @@ -448,7 +448,7 @@ grpc_channel_args* BuildXdsChannelArgs(const grpc_channel_args& args) { absl::InlinedVector args_to_add; // Keepalive interval. args_to_add.emplace_back(grpc_channel_arg_integer_create( - const_cast(GRPC_ARG_KEEPALIVE_TIME_MS), 5000)); + const_cast(GRPC_ARG_KEEPALIVE_TIME_MS), 5 * 60 * GPR_MS_PER_SEC)); // A channel arg indicating that the target is an xds server. // TODO(roth): Once we figure out our fallback and credentials story, decide // whether this is actually needed. Note that it's currently used by the From 9f7ff86919b29c806c4f3baa05d8bf69262de86b Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Thu, 27 Aug 2020 15:04:23 -0700 Subject: [PATCH 45/62] Accept the Android SDK licences. --- tools/internal_ci/linux/grpc_android.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/internal_ci/linux/grpc_android.sh b/tools/internal_ci/linux/grpc_android.sh index 81acbed6551..87c7a24dd7f 100755 --- a/tools/internal_ci/linux/grpc_android.sh +++ b/tools/internal_ci/linux/grpc_android.sh @@ -19,6 +19,9 @@ set -ex # Protobuf needs autoconf & automake to build sudo apt-get install -y autoconf automake +# Accept the Android SDK licences. +yes | /opt/android-sdk/current/tools/bin/sdkmanager --licenses + # change to grpc repo root cd $(dirname $0)/../../.. From 32ea9af3b0541f059f967dab0ad5634e73a95cb4 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Thu, 27 Aug 2020 16:44:27 -0700 Subject: [PATCH 46/62] Bump Android SDK --- examples/android/helloworld/build.gradle | 2 +- src/android/test/interop/app/build.gradle | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/android/helloworld/build.gradle b/examples/android/helloworld/build.gradle index bd5f337f013..78591e3b824 100644 --- a/examples/android/helloworld/build.gradle +++ b/examples/android/helloworld/build.gradle @@ -6,7 +6,7 @@ buildscript { jcenter() } dependencies { - classpath 'com.android.tools.build:gradle:3.0.1' + classpath 'com.android.tools.build:gradle:3.1.4' // NOTE: Do not place your application dependencies here; they belong // in the individual module build.gradle files } diff --git a/src/android/test/interop/app/build.gradle b/src/android/test/interop/app/build.gradle index fb500a71c71..5bedc43c9ef 100644 --- a/src/android/test/interop/app/build.gradle +++ b/src/android/test/interop/app/build.gradle @@ -29,6 +29,9 @@ android { arguments '-DgRPC_CPP_PLUGIN_EXECUTABLE=' + grpc_cpp_plugin } } + ndk { + abiFilters "armeabi-v7a", "arm64-v8a", "x86", "x86_64" + } } buildTypes { debug { From 7dcba1a700a2af129716783393af55d69a7b3f3e Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Fri, 21 Aug 2020 09:54:07 -0700 Subject: [PATCH 47/62] Make check_linker_need_libatomic more robust --- setup.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 20a6d76dbe0..88a7ef47956 100644 --- a/setup.py +++ b/setup.py @@ -171,7 +171,17 @@ def check_linker_need_libatomic(): stdout=PIPE, stderr=PIPE) cc_test.communicate(input=code_test) - return cc_test.returncode != 0 + if cc_test.returncode == 0: + return False + # Double-check to see if -latomic actually can solve the problem. + # https://github.com/grpc/grpc/issues/22491 + cc_test = subprocess.Popen( + ['cc', '-x', 'c++', '-std=c++11', '-latomic', '-'], + stdin=PIPE, + stdout=PIPE, + stderr=PIPE) + cc_test.communicate(input=code_test) + return cc_test.returncode == 0 # There are some situations (like on Windows) where CC, CFLAGS, and LDFLAGS are From 3b8044ae4ec5d9791881e59b0ea84ac71f897921 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Thu, 27 Aug 2020 19:57:02 -0700 Subject: [PATCH 48/62] Use c++ instead of cc --- setup.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/setup.py b/setup.py index 88a7ef47956..18be2e17817 100644 --- a/setup.py +++ b/setup.py @@ -166,22 +166,22 @@ def check_linker_need_libatomic(): """Test if linker on system needs libatomic.""" code_test = (b'#include \n' + b'int main() { return std::atomic{}; }') - cc_test = subprocess.Popen(['cc', '-x', 'c++', '-std=c++11', '-'], - stdin=PIPE, - stdout=PIPE, - stderr=PIPE) - cc_test.communicate(input=code_test) - if cc_test.returncode == 0: + cpp_test = subprocess.Popen(['c++', '-x', 'c++', '-std=c++11', '-'], + stdin=PIPE, + stdout=PIPE, + stderr=PIPE) + cpp_test.communicate(input=code_test) + if cpp_test.returncode == 0: return False # Double-check to see if -latomic actually can solve the problem. # https://github.com/grpc/grpc/issues/22491 - cc_test = subprocess.Popen( - ['cc', '-x', 'c++', '-std=c++11', '-latomic', '-'], + cpp_test = subprocess.Popen( + ['c++', '-x', 'c++', '-std=c++11', '-latomic', '-'], stdin=PIPE, stdout=PIPE, stderr=PIPE) - cc_test.communicate(input=code_test) - return cc_test.returncode == 0 + cpp_test.communicate(input=code_test) + return cpp_test.returncode == 0 # There are some situations (like on Windows) where CC, CFLAGS, and LDFLAGS are From 8d8152ed4ec89a98ee00b934809fc1aa0fb3f0b1 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 28 Aug 2020 13:58:24 +0200 Subject: [PATCH 49/62] fix System.Memory dependency mismatch in c# distribtests --- test/distrib/csharp/DistribTest/DistribTest.csproj | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/distrib/csharp/DistribTest/DistribTest.csproj b/test/distrib/csharp/DistribTest/DistribTest.csproj index e083d6225e7..f11f5c1e163 100644 --- a/test/distrib/csharp/DistribTest/DistribTest.csproj +++ b/test/distrib/csharp/DistribTest/DistribTest.csproj @@ -61,6 +61,10 @@ Use GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE to still allow building the protobuf generate code --> $(DefineConstants);GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + + true From 37c980d98a7b0a1acf4fbc4c388f4af034b4acec Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 28 Aug 2020 14:38:35 +0200 Subject: [PATCH 50/62] improve python3 compatibility of the task_runner.py script --- tools/run_tests/artifacts/distribtest_targets.py | 12 ++++++++++-- tools/run_tests/task_runner.py | 4 ++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/tools/run_tests/artifacts/distribtest_targets.py b/tools/run_tests/artifacts/distribtest_targets.py index 816729e3e8a..2684cda8c4d 100644 --- a/tools/run_tests/artifacts/distribtest_targets.py +++ b/tools/run_tests/artifacts/distribtest_targets.py @@ -235,7 +235,9 @@ class PHPDistribTest(object): self.platform = platform self.arch = arch self.docker_suffix = docker_suffix - self.labels = ['distribtest', 'php', platform, arch, docker_suffix] + self.labels = ['distribtest', 'php', platform, arch] + if docker_suffix: + self.labels.append(docker_suffix) def pre_build_jobspecs(self): return [] @@ -275,8 +277,14 @@ class CppDistribTest(object): self.docker_suffix = docker_suffix self.testcase = testcase self.labels = [ - 'distribtest', 'cpp', platform, arch, docker_suffix, testcase + 'distribtest', + 'cpp', + platform, + arch, + testcase, ] + if docker_suffix: + self.labels.append(docker_suffix) def pre_build_jobspecs(self): return [] diff --git a/tools/run_tests/task_runner.py b/tools/run_tests/task_runner.py index 067e11a861f..ab180046cb9 100755 --- a/tools/run_tests/task_runner.py +++ b/tools/run_tests/task_runner.py @@ -49,7 +49,7 @@ def _create_build_map(): if set(target_build_map.keys()).intersection(label_build_map.keys()): raise Exception('Target names need to be distinct from label names') - return dict(target_build_map.items() + label_build_map.items()) + return dict(list(target_build_map.items()) + list(label_build_map.items())) _BUILD_MAP = _create_build_map() @@ -83,7 +83,7 @@ for label in args.build: # Among targets selected by -b, filter out those that don't match the filter targets = [t for t in targets if all(f in t.labels for f in args.filter)] -targets = sorted(set(targets)) +targets = sorted(set(targets), key=lambda target: target.name) # Execute pre-build phase prebuild_jobs = [] From 30e4170dc36bef99afef95009b028754c9438232 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Fri, 28 Aug 2020 09:20:30 -0700 Subject: [PATCH 51/62] Update version of master to 1.33-dev ("geeky"). --- BUILD | 6 ++-- CMakeLists.txt | 10 +++--- Makefile | 32 +++++++++---------- build_config.rb | 2 +- build_handwritten.yaml | 6 ++-- doc/g_stands_for.md | 3 +- gRPC-C++.podspec | 2 +- gRPC-Core.podspec | 2 +- gRPC-ProtoRPC.podspec | 2 +- gRPC-RxLibrary.podspec | 2 +- gRPC.podspec | 2 +- package.xml | 6 ++-- src/core/lib/surface/version.cc | 4 +-- src/cpp/common/version_cc.cc | 2 +- src/csharp/Grpc.Core.Api/VersionInfo.cs | 4 +-- src/csharp/build/dependencies.props | 2 +- src/csharp/build_unitypackage.bat | 2 +- .../!ProtoCompiler-gRPCCppPlugin.podspec | 2 +- .../!ProtoCompiler-gRPCPlugin.podspec | 2 +- src/objective-c/GRPCClient/version.h | 2 +- src/objective-c/tests/version.h | 4 +-- src/php/composer.json | 2 +- src/php/ext/grpc/version.h | 2 +- src/python/grpcio/grpc/_grpcio_metadata.py | 2 +- src/python/grpcio/grpc_version.py | 2 +- src/python/grpcio_channelz/grpc_version.py | 2 +- .../grpcio_health_checking/grpc_version.py | 2 +- src/python/grpcio_reflection/grpc_version.py | 2 +- src/python/grpcio_status/grpc_version.py | 2 +- src/python/grpcio_testing/grpc_version.py | 2 +- src/python/grpcio_tests/grpc_version.py | 2 +- src/ruby/lib/grpc/version.rb | 2 +- src/ruby/tools/version.rb | 2 +- tools/distrib/python/grpc_version.py | 2 +- .../python/grpcio_tools/grpc_version.py | 2 +- tools/doxygen/Doxyfile.c++ | 2 +- tools/doxygen/Doxyfile.c++.internal | 2 +- tools/doxygen/Doxyfile.core | 2 +- tools/doxygen/Doxyfile.core.internal | 2 +- tools/doxygen/Doxyfile.objc | 2 +- tools/doxygen/Doxyfile.objc.internal | 2 +- tools/doxygen/Doxyfile.php | 2 +- 42 files changed, 71 insertions(+), 70 deletions(-) diff --git a/BUILD b/BUILD index 384061e14a2..4892aa92e26 100644 --- a/BUILD +++ b/BUILD @@ -75,11 +75,11 @@ config_setting( python_config_settings() # This should be updated along with build_handwritten.yaml -g_stands_for = "giggle" +g_stands_for = "geeky" -core_version = "11.0.0" +core_version = "12.0.0" -version = "1.32.0-dev" +version = "1.33.0-dev" GPR_PUBLIC_HDRS = [ "include/grpc/support/alloc.h", diff --git a/CMakeLists.txt b/CMakeLists.txt index 3b63d02f1e3..86ff4857183 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,12 +25,12 @@ cmake_minimum_required(VERSION 3.5.1) set(PACKAGE_NAME "grpc") -set(PACKAGE_VERSION "1.32.0-dev") -set(gRPC_CORE_VERSION "11.0.0") -set(gRPC_CORE_SOVERSION "11") -set(gRPC_CPP_VERSION "1.32.0-dev") +set(PACKAGE_VERSION "1.33.0-dev") +set(gRPC_CORE_VERSION "12.0.0") +set(gRPC_CORE_SOVERSION "12") +set(gRPC_CPP_VERSION "1.33.0-dev") set(gRPC_CPP_SOVERSION "1") -set(gRPC_CSHARP_VERSION "2.32.0-dev") +set(gRPC_CSHARP_VERSION "2.33.0-dev") set(gRPC_CSHARP_SOVERSION "2") set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}") set(PACKAGE_TARNAME "${PACKAGE_NAME}-${PACKAGE_VERSION}") diff --git a/Makefile b/Makefile index afeb11e1f3a..641762e7f06 100644 --- a/Makefile +++ b/Makefile @@ -456,9 +456,9 @@ E = @echo Q = @ endif -CORE_VERSION = 11.0.0 -CPP_VERSION = 1.32.0-dev -CSHARP_VERSION = 2.32.0-dev +CORE_VERSION = 12.0.0 +CPP_VERSION = 1.33.0-dev +CSHARP_VERSION = 2.33.0-dev CPPFLAGS_NO_ARCH += $(addprefix -I, $(INCLUDES)) $(addprefix -D, $(DEFINES)) CPPFLAGS += $(CPPFLAGS_NO_ARCH) $(ARCH_FLAGS) @@ -500,7 +500,7 @@ SHARED_EXT_CORE = dll SHARED_EXT_CPP = dll SHARED_EXT_CSHARP = dll SHARED_PREFIX = -SHARED_VERSION_CORE = -11 +SHARED_VERSION_CORE = -12 SHARED_VERSION_CPP = -1 SHARED_VERSION_CSHARP = -2 else ifeq ($(SYSTEM),Darwin) @@ -1872,8 +1872,8 @@ $(LIBDIR)/$(CONFIG)/libaddress_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): ifeq ($(SYSTEM),Darwin) $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)address_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libaddress_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBADDRESS_SORTING_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(RE2_MERGE_LIBS) $(UPB_MERGE_LIBS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) else - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libaddress_sorting.so.11 -o $(LIBDIR)/$(CONFIG)/libaddress_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBADDRESS_SORTING_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(RE2_MERGE_LIBS) $(UPB_MERGE_LIBS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) - $(Q) ln -sf $(SHARED_PREFIX)address_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libaddress_sorting$(SHARED_VERSION_CORE).so.11 + $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libaddress_sorting.so.12 -o $(LIBDIR)/$(CONFIG)/libaddress_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBADDRESS_SORTING_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(RE2_MERGE_LIBS) $(UPB_MERGE_LIBS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) + $(Q) ln -sf $(SHARED_PREFIX)address_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libaddress_sorting$(SHARED_VERSION_CORE).so.12 $(Q) ln -sf $(SHARED_PREFIX)address_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libaddress_sorting$(SHARED_VERSION_CORE).so endif endif @@ -1997,8 +1997,8 @@ $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBGPR_OB ifeq ($(SYSTEM),Darwin) $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGPR_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(RE2_MERGE_LIBS) $(UPB_MERGE_LIBS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) else - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgpr.so.11 -o $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGPR_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(RE2_MERGE_LIBS) $(UPB_MERGE_LIBS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) - $(Q) ln -sf $(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).so.11 + $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgpr.so.12 -o $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGPR_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(RE2_MERGE_LIBS) $(UPB_MERGE_LIBS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) + $(Q) ln -sf $(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).so.12 $(Q) ln -sf $(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).so endif endif @@ -2490,8 +2490,8 @@ $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBGRPC_ ifeq ($(SYSTEM),Darwin) $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_OBJS) $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a $(OPENSSL_MERGE_LIBS) $(LDLIBS_SECURE) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(RE2_MERGE_LIBS) $(UPB_MERGE_LIBS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) else - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgrpc.so.11 -o $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_OBJS) $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a $(OPENSSL_MERGE_LIBS) $(LDLIBS_SECURE) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(RE2_MERGE_LIBS) $(UPB_MERGE_LIBS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) - $(Q) ln -sf $(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).so.11 + $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgrpc.so.12 -o $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_OBJS) $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a $(OPENSSL_MERGE_LIBS) $(LDLIBS_SECURE) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(RE2_MERGE_LIBS) $(UPB_MERGE_LIBS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) + $(Q) ln -sf $(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).so.12 $(Q) ln -sf $(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).so endif endif @@ -2549,8 +2549,8 @@ $(LIBDIR)/$(CONFIG)/libgrpc_csharp_ext$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): ifeq ($(SYSTEM),Darwin) $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)grpc_csharp_ext$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libgrpc_csharp_ext$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_CSHARP_EXT_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(RE2_MERGE_LIBS) $(UPB_MERGE_LIBS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) else - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgrpc_csharp_ext.so.11 -o $(LIBDIR)/$(CONFIG)/libgrpc_csharp_ext$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_CSHARP_EXT_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(RE2_MERGE_LIBS) $(UPB_MERGE_LIBS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) - $(Q) ln -sf $(SHARED_PREFIX)grpc_csharp_ext$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc_csharp_ext$(SHARED_VERSION_CORE).so.11 + $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgrpc_csharp_ext.so.12 -o $(LIBDIR)/$(CONFIG)/libgrpc_csharp_ext$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_CSHARP_EXT_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(RE2_MERGE_LIBS) $(UPB_MERGE_LIBS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) + $(Q) ln -sf $(SHARED_PREFIX)grpc_csharp_ext$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc_csharp_ext$(SHARED_VERSION_CORE).so.12 $(Q) ln -sf $(SHARED_PREFIX)grpc_csharp_ext$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc_csharp_ext$(SHARED_VERSION_CORE).so endif endif @@ -2958,8 +2958,8 @@ $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $ ifeq ($(SYSTEM),Darwin) $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(RE2_MERGE_LIBS) $(UPB_MERGE_LIBS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) else - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgrpc_unsecure.so.11 -o $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(RE2_MERGE_LIBS) $(UPB_MERGE_LIBS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) - $(Q) ln -sf $(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).so.11 + $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgrpc_unsecure.so.12 -o $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(RE2_MERGE_LIBS) $(UPB_MERGE_LIBS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) + $(Q) ln -sf $(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).so.12 $(Q) ln -sf $(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).so endif endif @@ -4305,8 +4305,8 @@ $(LIBDIR)/$(CONFIG)/libupb$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBUPB_OB ifeq ($(SYSTEM),Darwin) $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)upb$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libupb$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUPB_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(RE2_MERGE_LIBS) $(UPB_MERGE_LIBS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) else - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libupb.so.11 -o $(LIBDIR)/$(CONFIG)/libupb$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUPB_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(RE2_MERGE_LIBS) $(UPB_MERGE_LIBS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) - $(Q) ln -sf $(SHARED_PREFIX)upb$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libupb$(SHARED_VERSION_CORE).so.11 + $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libupb.so.12 -o $(LIBDIR)/$(CONFIG)/libupb$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUPB_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(RE2_MERGE_LIBS) $(UPB_MERGE_LIBS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) + $(Q) ln -sf $(SHARED_PREFIX)upb$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libupb$(SHARED_VERSION_CORE).so.12 $(Q) ln -sf $(SHARED_PREFIX)upb$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libupb$(SHARED_VERSION_CORE).so endif endif diff --git a/build_config.rb b/build_config.rb index ff432bd3392..642399878ae 100644 --- a/build_config.rb +++ b/build_config.rb @@ -13,5 +13,5 @@ # limitations under the License. module GrpcBuildConfig - CORE_WINDOWS_DLL = '/tmp/libs/opt/grpc-11.dll' + CORE_WINDOWS_DLL = '/tmp/libs/opt/grpc-12.dll' end diff --git a/build_handwritten.yaml b/build_handwritten.yaml index 5c45f8e29a4..4a4263bc273 100644 --- a/build_handwritten.yaml +++ b/build_handwritten.yaml @@ -12,10 +12,10 @@ settings: '#08': Use "-preN" suffixes to identify pre-release versions '#09': Per-language overrides are possible with (eg) ruby_version tag here '#10': See the expand_version.py for all the quirks here - core_version: 11.0.0 + core_version: 12.0.0 csharp_major_version: 2 - g_stands_for: giggle - version: 1.32.0-dev + g_stands_for: geeky + version: 1.33.0-dev targets: - name: check_epollexclusive build: tool diff --git a/doc/g_stands_for.md b/doc/g_stands_for.md index 069d4e5a609..c384913de20 100644 --- a/doc/g_stands_for.md +++ b/doc/g_stands_for.md @@ -31,4 +31,5 @@ - 1.29 'g' stands for ['gringotts'](https://github.com/grpc/grpc/tree/v1.29.x) - 1.30 'g' stands for ['gradius'](https://github.com/grpc/grpc/tree/v1.30.x) - 1.31 'g' stands for ['galore'](https://github.com/grpc/grpc/tree/v1.31.x) -- 1.32 'g' stands for ['giggle'](https://github.com/grpc/grpc/tree/master) +- 1.32 'g' stands for ['giggle'](https://github.com/grpc/grpc/tree/v1.32.x) +- 1.33 'g' stands for ['geeky'](https://github.com/grpc/grpc/tree/master) diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index 42bb53c9fe3..2ba4d53a57b 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -22,7 +22,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-C++' # TODO (mxyan): use version that match gRPC version when pod is stabilized - version = '1.32.0-dev' + version = '1.33.0-dev' s.version = version s.summary = 'gRPC C++ library' s.homepage = 'https://grpc.io' diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 558884b7226..db4a2ff0a44 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-Core' - version = '1.32.0-dev' + version = '1.33.0-dev' s.version = version s.summary = 'Core cross-platform gRPC library, written in C' s.homepage = 'https://grpc.io' diff --git a/gRPC-ProtoRPC.podspec b/gRPC-ProtoRPC.podspec index 705e5a850b4..bcc6ffe249c 100644 --- a/gRPC-ProtoRPC.podspec +++ b/gRPC-ProtoRPC.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-ProtoRPC' - version = '1.32.0-dev' + version = '1.33.0-dev' s.version = version s.summary = 'RPC library for Protocol Buffers, based on gRPC' s.homepage = 'https://grpc.io' diff --git a/gRPC-RxLibrary.podspec b/gRPC-RxLibrary.podspec index 9b1eab1b4b9..83fff1d66f2 100644 --- a/gRPC-RxLibrary.podspec +++ b/gRPC-RxLibrary.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-RxLibrary' - version = '1.32.0-dev' + version = '1.33.0-dev' s.version = version s.summary = 'Reactive Extensions library for iOS/OSX.' s.homepage = 'https://grpc.io' diff --git a/gRPC.podspec b/gRPC.podspec index 1df3cce0057..70557e55134 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -20,7 +20,7 @@ Pod::Spec.new do |s| s.name = 'gRPC' - version = '1.32.0-dev' + version = '1.33.0-dev' s.version = version s.summary = 'gRPC client library for iOS/OSX' s.homepage = 'https://grpc.io' diff --git a/package.xml b/package.xml index 5c3d26ca57c..c896bbe0f2d 100644 --- a/package.xml +++ b/package.xml @@ -13,8 +13,8 @@ 2019-09-24 - 1.32.0dev - 1.32.0dev + 1.33.0dev + 1.33.0dev beta @@ -22,7 +22,7 @@ Apache 2.0 -- gRPC Core 1.32.0 update +- gRPC Core 1.33.0 update diff --git a/src/core/lib/surface/version.cc b/src/core/lib/surface/version.cc index 181a480d955..e022da47a69 100644 --- a/src/core/lib/surface/version.cc +++ b/src/core/lib/surface/version.cc @@ -23,6 +23,6 @@ #include -const char* grpc_version_string(void) { return "11.0.0"; } +const char* grpc_version_string(void) { return "12.0.0"; } -const char* grpc_g_stands_for(void) { return "giggle"; } +const char* grpc_g_stands_for(void) { return "geeky"; } diff --git a/src/cpp/common/version_cc.cc b/src/cpp/common/version_cc.cc index e37246bcfa0..479c4fa228f 100644 --- a/src/cpp/common/version_cc.cc +++ b/src/cpp/common/version_cc.cc @@ -22,5 +22,5 @@ #include namespace grpc { -std::string Version() { return "1.32.0-dev"; } +std::string Version() { return "1.33.0-dev"; } } // namespace grpc diff --git a/src/csharp/Grpc.Core.Api/VersionInfo.cs b/src/csharp/Grpc.Core.Api/VersionInfo.cs index 0c7416b0c6e..feaf39ccdf0 100644 --- a/src/csharp/Grpc.Core.Api/VersionInfo.cs +++ b/src/csharp/Grpc.Core.Api/VersionInfo.cs @@ -33,11 +33,11 @@ namespace Grpc.Core /// /// Current AssemblyFileVersion of gRPC C# assemblies /// - public const string CurrentAssemblyFileVersion = "2.32.0.0"; + public const string CurrentAssemblyFileVersion = "2.33.0.0"; /// /// Current version of gRPC C# /// - public const string CurrentVersion = "2.32.0-dev"; + public const string CurrentVersion = "2.33.0-dev"; } } diff --git a/src/csharp/build/dependencies.props b/src/csharp/build/dependencies.props index 5e6658b29b7..9fcb122009e 100644 --- a/src/csharp/build/dependencies.props +++ b/src/csharp/build/dependencies.props @@ -1,7 +1,7 @@ - 2.32.0-dev + 2.33.0-dev 3.13.0 diff --git a/src/csharp/build_unitypackage.bat b/src/csharp/build_unitypackage.bat index 8915ebdf8fc..1568b6d2789 100644 --- a/src/csharp/build_unitypackage.bat +++ b/src/csharp/build_unitypackage.bat @@ -13,7 +13,7 @@ @rem limitations under the License. @rem Current package versions -set VERSION=2.32.0-dev +set VERSION=2.33.0-dev @rem Adjust the location of nuget.exe set NUGET=C:\nuget\nuget.exe diff --git a/src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec index b00ae58875e..71c71503033 100644 --- a/src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec @@ -42,7 +42,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler-gRPCCppPlugin' - v = '1.32.0-dev' + v = '1.33.0-dev' s.version = v s.summary = 'The gRPC ProtoC plugin generates C++ files from .proto services.' s.description = <<-DESC diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index 48d2f7abd35..bdb3a66de1e 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -42,7 +42,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler-gRPCPlugin' - v = '1.32.0-dev' + v = '1.33.0-dev' s.version = v s.summary = 'The gRPC ProtoC plugin generates Objective-C files from .proto services.' s.description = <<-DESC diff --git a/src/objective-c/GRPCClient/version.h b/src/objective-c/GRPCClient/version.h index acf091a3c51..d9e73d2eb4b 100644 --- a/src/objective-c/GRPCClient/version.h +++ b/src/objective-c/GRPCClient/version.h @@ -22,4 +22,4 @@ // instead. This file can be regenerated from the template by running // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.32.0-dev" +#define GRPC_OBJC_VERSION_STRING @"1.33.0-dev" diff --git a/src/objective-c/tests/version.h b/src/objective-c/tests/version.h index b16c890372d..902692831b3 100644 --- a/src/objective-c/tests/version.h +++ b/src/objective-c/tests/version.h @@ -22,5 +22,5 @@ // instead. This file can be regenerated from the template by running // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.32.0-dev" -#define GRPC_C_VERSION_STRING @"11.0.0" +#define GRPC_OBJC_VERSION_STRING @"1.33.0-dev" +#define GRPC_C_VERSION_STRING @"12.0.0" diff --git a/src/php/composer.json b/src/php/composer.json index d69036895ef..df51d7aac20 100644 --- a/src/php/composer.json +++ b/src/php/composer.json @@ -2,7 +2,7 @@ "name": "grpc/grpc-dev", "description": "gRPC library for PHP - for Development use only", "license": "Apache-2.0", - "version": "1.32.0", + "version": "1.33.0", "require": { "php": ">=5.5.0", "google/protobuf": "^v3.3.0" diff --git a/src/php/ext/grpc/version.h b/src/php/ext/grpc/version.h index 777b3f0f9d4..55111a5ad0b 100644 --- a/src/php/ext/grpc/version.h +++ b/src/php/ext/grpc/version.h @@ -20,6 +20,6 @@ #ifndef VERSION_H #define VERSION_H -#define PHP_GRPC_VERSION "1.32.0dev" +#define PHP_GRPC_VERSION "1.33.0dev" #endif /* VERSION_H */ diff --git a/src/python/grpcio/grpc/_grpcio_metadata.py b/src/python/grpcio/grpc/_grpcio_metadata.py index 4f2c71a51b5..fe2b7f5879d 100644 --- a/src/python/grpcio/grpc/_grpcio_metadata.py +++ b/src/python/grpcio/grpc/_grpcio_metadata.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc/_grpcio_metadata.py.template`!!! -__version__ = """1.32.0.dev0""" +__version__ = """1.33.0.dev0""" diff --git a/src/python/grpcio/grpc_version.py b/src/python/grpcio/grpc_version.py index 55270b0f447..036c15b0ecb 100644 --- a/src/python/grpcio/grpc_version.py +++ b/src/python/grpcio/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_version.py.template`!!! -VERSION = '1.32.0.dev0' +VERSION = '1.33.0.dev0' diff --git a/src/python/grpcio_channelz/grpc_version.py b/src/python/grpcio_channelz/grpc_version.py index afc27a8bfff..85bece3c009 100644 --- a/src/python/grpcio_channelz/grpc_version.py +++ b/src/python/grpcio_channelz/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_channelz/grpc_version.py.template`!!! -VERSION = '1.32.0.dev0' +VERSION = '1.33.0.dev0' diff --git a/src/python/grpcio_health_checking/grpc_version.py b/src/python/grpcio_health_checking/grpc_version.py index 90b465cdf25..815fff2a3f8 100644 --- a/src/python/grpcio_health_checking/grpc_version.py +++ b/src/python/grpcio_health_checking/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_health_checking/grpc_version.py.template`!!! -VERSION = '1.32.0.dev0' +VERSION = '1.33.0.dev0' diff --git a/src/python/grpcio_reflection/grpc_version.py b/src/python/grpcio_reflection/grpc_version.py index cf587708391..2dd4b7403e0 100644 --- a/src/python/grpcio_reflection/grpc_version.py +++ b/src/python/grpcio_reflection/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_reflection/grpc_version.py.template`!!! -VERSION = '1.32.0.dev0' +VERSION = '1.33.0.dev0' diff --git a/src/python/grpcio_status/grpc_version.py b/src/python/grpcio_status/grpc_version.py index b75a7215c76..03ef182a288 100644 --- a/src/python/grpcio_status/grpc_version.py +++ b/src/python/grpcio_status/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_status/grpc_version.py.template`!!! -VERSION = '1.32.0.dev0' +VERSION = '1.33.0.dev0' diff --git a/src/python/grpcio_testing/grpc_version.py b/src/python/grpcio_testing/grpc_version.py index 2068e2b5a1b..c74f56c17cf 100644 --- a/src/python/grpcio_testing/grpc_version.py +++ b/src/python/grpcio_testing/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_testing/grpc_version.py.template`!!! -VERSION = '1.32.0.dev0' +VERSION = '1.33.0.dev0' diff --git a/src/python/grpcio_tests/grpc_version.py b/src/python/grpcio_tests/grpc_version.py index fbff270f918..82e01e25915 100644 --- a/src/python/grpcio_tests/grpc_version.py +++ b/src/python/grpcio_tests/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_tests/grpc_version.py.template`!!! -VERSION = '1.32.0.dev0' +VERSION = '1.33.0.dev0' diff --git a/src/ruby/lib/grpc/version.rb b/src/ruby/lib/grpc/version.rb index 3e81b021c3b..daabad9e9a4 100644 --- a/src/ruby/lib/grpc/version.rb +++ b/src/ruby/lib/grpc/version.rb @@ -14,5 +14,5 @@ # GRPC contains the General RPC module. module GRPC - VERSION = '1.32.0.dev' + VERSION = '1.33.0.dev' end diff --git a/src/ruby/tools/version.rb b/src/ruby/tools/version.rb index 778eeacb68a..c21abf825e6 100644 --- a/src/ruby/tools/version.rb +++ b/src/ruby/tools/version.rb @@ -14,6 +14,6 @@ module GRPC module Tools - VERSION = '1.32.0.dev' + VERSION = '1.33.0.dev' end end diff --git a/tools/distrib/python/grpc_version.py b/tools/distrib/python/grpc_version.py index 3f85118019d..fec9dadecf4 100644 --- a/tools/distrib/python/grpc_version.py +++ b/tools/distrib/python/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION = '1.32.0.dev0' +VERSION = '1.33.0.dev0' diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index 0b7c753bdfd..54a1dd53902 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION = '1.32.0.dev0' +VERSION = '1.33.0.dev0' diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index af4ca9e6547..425da810996 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.32.0-dev +PROJECT_NUMBER = 1.33.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 5876739a4b8..5d601ba7c13 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.32.0-dev +PROJECT_NUMBER = 1.33.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index 8da50b0c21b..dc51def7ed2 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC Core" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 11.0.0 +PROJECT_NUMBER = 12.0.0 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index c663f08e41f..3b00ee9aba0 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC Core" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 11.0.0 +PROJECT_NUMBER = 12.0.0 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.objc b/tools/doxygen/Doxyfile.objc index 5ae4647fd22..5bcec39f1d8 100644 --- a/tools/doxygen/Doxyfile.objc +++ b/tools/doxygen/Doxyfile.objc @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC Objective-C" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.32.0-dev +PROJECT_NUMBER = 1.33.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.objc.internal b/tools/doxygen/Doxyfile.objc.internal index 636c73f0660..1c8424a48b0 100644 --- a/tools/doxygen/Doxyfile.objc.internal +++ b/tools/doxygen/Doxyfile.objc.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC Objective-C" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.32.0-dev +PROJECT_NUMBER = 1.33.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.php b/tools/doxygen/Doxyfile.php index c1025e5cf55..097f40fb947 100644 --- a/tools/doxygen/Doxyfile.php +++ b/tools/doxygen/Doxyfile.php @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC PHP" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.32.0-dev +PROJECT_NUMBER = 1.33.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a From 5c1ebcbb40b00af46f91e0a90ae884c14de046b9 Mon Sep 17 00:00:00 2001 From: yulin-liang Date: Wed, 26 Aug 2020 22:25:06 -0700 Subject: [PATCH 52/62] Fix flaky macos/grpc_cfstream test by extending NTP sync timeout. --- tools/internal_ci/macos/grpc_run_bazel_isolated_tests.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/internal_ci/macos/grpc_run_bazel_isolated_tests.sh b/tools/internal_ci/macos/grpc_run_bazel_isolated_tests.sh index d4d3535d8c7..78ea9b6facb 100644 --- a/tools/internal_ci/macos/grpc_run_bazel_isolated_tests.sh +++ b/tools/internal_ci/macos/grpc_run_bazel_isolated_tests.sh @@ -24,6 +24,12 @@ cd $(dirname $0)/../../.. # The "local" execution strategy is required because the test runs sudo and that doesn't work in a sandboxed environment (the default on mac) tools/bazel test $RUN_TESTS_FLAGS --spawn_strategy=local --genrule_strategy=local --test_output=all --copt="-DGRPC_CFSTREAM=1" //test/cpp/end2end:cfstream_test +# Missing the /var/db/ntp-kod file may breaks the ntp synchronization. +# Create the file and change the ownership to root before NTP sync. +# TODO(yulin-liang): investigate how to run time_jump_test without needing to mess with the system time directly. +# See b/166245303 +sudo touch /var/db/ntp-kod +sudo chown root:wheel /var/db/ntp-kod # Make sure time is in sync before running time_jump_test because the test does # NTP sync before exiting. Bazel gets confused if test end time < start time. sudo sntp -sS pool.ntp.org From fc4556a92d6a300bdac3bf182a2979680c499837 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Thu, 27 Aug 2020 20:34:58 -0700 Subject: [PATCH 53/62] Add ABSL_NO_XRAY_ATTRIBUTES --- examples/android/helloworld/app/CMakeLists.txt | 3 +++ src/android/test/interop/app/CMakeLists.txt | 3 +++ 2 files changed, 6 insertions(+) diff --git a/examples/android/helloworld/app/CMakeLists.txt b/examples/android/helloworld/app/CMakeLists.txt index f449a5ce2ea..160eb845d99 100644 --- a/examples/android/helloworld/app/CMakeLists.txt +++ b/examples/android/helloworld/app/CMakeLists.txt @@ -2,6 +2,9 @@ cmake_minimum_required(VERSION 3.4.1) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") +# https://github.com/abseil/abseil-cpp/issues/626 +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DABSL_NO_XRAY_ATTRIBUTES=1") + set(helloworld_PROTOBUF_PROTOC_EXECUTABLE "/usr/local/bin/protoc" CACHE STRING "Protoc binary on host") set(helloworld_GRPC_CPP_PLUGIN_EXECUTABLE "/usr/local/bin/grpc_cpp_plugin" CACHE STRING "gRPC CPP plugin binary on host") diff --git a/src/android/test/interop/app/CMakeLists.txt b/src/android/test/interop/app/CMakeLists.txt index e288dd63cec..53fbe594499 100644 --- a/src/android/test/interop/app/CMakeLists.txt +++ b/src/android/test/interop/app/CMakeLists.txt @@ -2,6 +2,9 @@ cmake_minimum_required(VERSION 3.4.1) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") +# https://github.com/abseil/abseil-cpp/issues/626 +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DABSL_NO_XRAY_ATTRIBUTES=1") + set(PROTOBUF_PROTOC_EXECUTABLE "/usr/local/bin/protoc" CACHE STRING "Protoc binary on host") set(gRPC_CPP_PLUGIN_EXECUTABLE "/usr/local/bin/grpc_cpp_plugin" CACHE STRING "gRPC CPP plugin binary on host") From 6bce82c42ad205d8a6b6df3af210d57b4738d1a4 Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Fri, 28 Aug 2020 12:26:14 -0700 Subject: [PATCH 54/62] Increase wait for stats timeout --- tools/run_tests/run_xds_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/run_xds_tests.py b/tools/run_tests/run_xds_tests.py index 285658308f1..9d2d8527b28 100755 --- a/tools/run_tests/run_xds_tests.py +++ b/tools/run_tests/run_xds_tests.py @@ -226,7 +226,7 @@ _WAIT_FOR_BACKEND_SEC = args.wait_for_backend_sec _WAIT_FOR_OPERATION_SEC = 1200 _INSTANCE_GROUP_SIZE = args.instance_group_size _NUM_TEST_RPCS = 10 * args.qps -_WAIT_FOR_STATS_SEC = 180 +_WAIT_FOR_STATS_SEC = 360 _WAIT_FOR_VALID_CONFIG_SEC = 60 _WAIT_FOR_URL_MAP_PATCH_SEC = 300 _CONNECTION_TIMEOUT_SEC = 60 From 03ba9336dd97831ce9ca740ffa27863cf3dd3532 Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Fri, 28 Aug 2020 13:13:00 -0700 Subject: [PATCH 55/62] Increase xds job timeout for python and c# --- tools/internal_ci/linux/grpc_xds_csharp.cfg | 2 +- tools/internal_ci/linux/grpc_xds_python.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/internal_ci/linux/grpc_xds_csharp.cfg b/tools/internal_ci/linux/grpc_xds_csharp.cfg index 579cd47277c..1f6ab280ad0 100644 --- a/tools/internal_ci/linux/grpc_xds_csharp.cfg +++ b/tools/internal_ci/linux/grpc_xds_csharp.cfg @@ -16,7 +16,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/linux/grpc_xds_csharp.sh" -timeout_mins: 90 +timeout_mins: 120 action { define_artifacts { regex: "**/*sponge_log.*" diff --git a/tools/internal_ci/linux/grpc_xds_python.cfg b/tools/internal_ci/linux/grpc_xds_python.cfg index 922b1aa88a1..8eea6fe3831 100644 --- a/tools/internal_ci/linux/grpc_xds_python.cfg +++ b/tools/internal_ci/linux/grpc_xds_python.cfg @@ -16,7 +16,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/linux/grpc_xds.sh" -timeout_mins: 90 +timeout_mins: 120 env_vars { key: "BAZEL_SCRIPT" value: "tools/internal_ci/linux/grpc_xds_bazel_python_test_in_docker.sh" From 4df4ea586fee2223519500470dee6f94602bf9fc Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Fri, 28 Aug 2020 13:33:45 -0700 Subject: [PATCH 56/62] remove new instance group spec, redundant with remove instance group --- doc/xds-test-descriptions.md | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/doc/xds-test-descriptions.md b/doc/xds-test-descriptions.md index c42583b2010..06c1d50af54 100644 --- a/doc/xds-test-descriptions.md +++ b/doc/xds-test-descriptions.md @@ -218,31 +218,6 @@ Test driver asserts: 1. All backends in the primary locality receive at least 1 RPC. 1. No backends in the secondary locality receive RPCs. -### new_instance_group_receives_traffic - -This test verifies that new instance groups added to a backend service in the -same zone receive traffic. - -Client parameters: - -1. --num_channels=1 -1. --qps=100 -1. --fail_on_failed_rpc=true - -Load balancer configuration: - -1. One MIG with two backends, using rate balancing mode. - -Test driver asserts: - -1. All backends receive at least one RPC. - -The test driver adds a new MIG with two backends in the same zone. - -Test driver asserts: - -1. All backends in each MIG receive at least one RPC. - ### remove_instance_group This test verifies that a remaining instance group can successfully serve RPCs From 91dc46c59f1f87b70fa487a8333070b4821923ec Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Fri, 28 Aug 2020 13:36:55 -0700 Subject: [PATCH 57/62] add gentle_failover spec --- doc/xds-test-descriptions.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/doc/xds-test-descriptions.md b/doc/xds-test-descriptions.md index 06c1d50af54..721b1e7414c 100644 --- a/doc/xds-test-descriptions.md +++ b/doc/xds-test-descriptions.md @@ -296,3 +296,36 @@ Assert: 1. Once all backends receive at least one RPC, the following 1000 RPCs are distributed across the 2 backends as a: 20, b: 80. +### gentle_failover + +This test verifies that traffic is partially diverted to a secondary locality +when > 50% of the instances in the primary locality are unhealthy. + +Client parameters: + +1. --num_channels=1 +1. --qps=100 + +Load balancer configuration: + +1. The primary MIG with 3 backends in the same zone as the client +1. The secondary MIG with 2 backends in a different zone + +Test driver asserts: + +1. All backends in the primary locality receive at least 1 RPC. +1. No backends in the secondary locality receive RPCs. + +The test driver stops 2 of 3 backends in the primary locality. + +Test driver asserts: + +1. All backends in the secondary locality receive at least 1 RPC. +1. The remaining backend in the primary locality receives at least 1 RPC. + +The test driver resumes the backends in the primary locality. + +Test driver asserts: + +1. All backends in the primary locality receive at least 1 RPC. +1. No backends in the secondary locality receive RPCs. From 2a72f72a84c6d3d7f34e7481a39e1d4623c9fd44 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Fri, 28 Aug 2020 15:53:20 -0700 Subject: [PATCH 58/62] CertificateProvider API --- BUILD | 1 + BUILD.gn | 1 + build_autogenerated.yaml | 1 + gRPC-C++.podspec | 2 + gRPC-Core.podspec | 2 + grpc.gemspec | 1 + package.xml | 1 + src/core/lib/security/certificate_provider.h | 66 ++++++++++++++++++++ tools/doxygen/Doxyfile.c++.internal | 1 + tools/doxygen/Doxyfile.core.internal | 1 + 10 files changed, 77 insertions(+) create mode 100644 src/core/lib/security/certificate_provider.h diff --git a/BUILD b/BUILD index 4892aa92e26..2ff563e1048 100644 --- a/BUILD +++ b/BUILD @@ -1811,6 +1811,7 @@ grpc_cc_library( hdrs = [ "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h", "src/core/ext/xds/xds_channel_args.h", + "src/core/lib/security/certificate_provider.h", "src/core/lib/security/context/security_context.h", "src/core/lib/security/credentials/alts/alts_credentials.h", "src/core/lib/security/credentials/composite/composite_credentials.h", diff --git a/BUILD.gn b/BUILD.gn index 7992a11f1ee..2f193cf4bc9 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -799,6 +799,7 @@ config("grpc_config") { "src/core/lib/security/authorization/mock_cel/evaluator_core.h", "src/core/lib/security/authorization/mock_cel/flat_expr_builder.h", "src/core/lib/security/authorization/mock_cel/statusor.h", + "src/core/lib/security/certificate_provider.h", "src/core/lib/security/context/security_context.cc", "src/core/lib/security/context/security_context.h", "src/core/lib/security/credentials/alts/alts_credentials.cc", diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index aea2c9bcd51..58f4c19e20d 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -660,6 +660,7 @@ libs: - src/core/lib/security/authorization/mock_cel/evaluator_core.h - src/core/lib/security/authorization/mock_cel/flat_expr_builder.h - src/core/lib/security/authorization/mock_cel/statusor.h + - src/core/lib/security/certificate_provider.h - src/core/lib/security/context/security_context.h - src/core/lib/security/credentials/alts/alts_credentials.h - src/core/lib/security/credentials/alts/check_gcp_environment.h diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index 2ba4d53a57b..308c56a7b50 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -523,6 +523,7 @@ Pod::Spec.new do |s| 'src/core/lib/security/authorization/mock_cel/evaluator_core.h', 'src/core/lib/security/authorization/mock_cel/flat_expr_builder.h', 'src/core/lib/security/authorization/mock_cel/statusor.h', + 'src/core/lib/security/certificate_provider.h', 'src/core/lib/security/context/security_context.h', 'src/core/lib/security/credentials/alts/alts_credentials.h', 'src/core/lib/security/credentials/alts/check_gcp_environment.h', @@ -1022,6 +1023,7 @@ Pod::Spec.new do |s| 'src/core/lib/security/authorization/mock_cel/evaluator_core.h', 'src/core/lib/security/authorization/mock_cel/flat_expr_builder.h', 'src/core/lib/security/authorization/mock_cel/statusor.h', + 'src/core/lib/security/certificate_provider.h', 'src/core/lib/security/context/security_context.h', 'src/core/lib/security/credentials/alts/alts_credentials.h', 'src/core/lib/security/credentials/alts/check_gcp_environment.h', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index db4a2ff0a44..efa2d132f29 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -854,6 +854,7 @@ Pod::Spec.new do |s| 'src/core/lib/security/authorization/mock_cel/evaluator_core.h', 'src/core/lib/security/authorization/mock_cel/flat_expr_builder.h', 'src/core/lib/security/authorization/mock_cel/statusor.h', + 'src/core/lib/security/certificate_provider.h', 'src/core/lib/security/context/security_context.cc', 'src/core/lib/security/context/security_context.h', 'src/core/lib/security/credentials/alts/alts_credentials.cc', @@ -1433,6 +1434,7 @@ Pod::Spec.new do |s| 'src/core/lib/security/authorization/mock_cel/evaluator_core.h', 'src/core/lib/security/authorization/mock_cel/flat_expr_builder.h', 'src/core/lib/security/authorization/mock_cel/statusor.h', + 'src/core/lib/security/certificate_provider.h', 'src/core/lib/security/context/security_context.h', 'src/core/lib/security/credentials/alts/alts_credentials.h', 'src/core/lib/security/credentials/alts/check_gcp_environment.h', diff --git a/grpc.gemspec b/grpc.gemspec index 698e6dad6ed..116cfc602b4 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -772,6 +772,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/security/authorization/mock_cel/evaluator_core.h ) s.files += %w( src/core/lib/security/authorization/mock_cel/flat_expr_builder.h ) s.files += %w( src/core/lib/security/authorization/mock_cel/statusor.h ) + s.files += %w( src/core/lib/security/certificate_provider.h ) s.files += %w( src/core/lib/security/context/security_context.cc ) s.files += %w( src/core/lib/security/context/security_context.h ) s.files += %w( src/core/lib/security/credentials/alts/alts_credentials.cc ) diff --git a/package.xml b/package.xml index c896bbe0f2d..66bcccb9f69 100644 --- a/package.xml +++ b/package.xml @@ -752,6 +752,7 @@ + diff --git a/src/core/lib/security/certificate_provider.h b/src/core/lib/security/certificate_provider.h new file mode 100644 index 00000000000..48a37616de0 --- /dev/null +++ b/src/core/lib/security/certificate_provider.h @@ -0,0 +1,66 @@ +// +// +// Copyright 2020 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// + +#ifndef GRPC_CORE_LIB_SECURITY_CERTIFICATE_PROVIDER_H +#define GRPC_CORE_LIB_SECURITY_CERTIFICATE_PROVIDER_H + +#include + +#include "src/core/lib/gprpp/ref_counted_ptr.h" +#include "src/core/lib/iomgr/pollset_set.h" + +namespace grpc_core { + +// TODO(yashkt): After https://github.com/grpc/grpc/pull/23572, remove this +// forward declaration and include the header for the distributor instead. +struct grpc_tls_certificate_distributor; + +// Interface for a grpc_tls_certificate_provider that handles the process to +// fetch credentials and validation contexts. Implementations are free to rely +// on local or remote sources to fetch the latest secrets, and free to share any +// state among different instances as they deem fit. +// +// On creation, grpc_tls_certificate_provider creates a +// grpc_tls_certificate_distributor object. When the credentials and validation +// contexts become valid or changed, a grpc_tls_certificate_provider should +// notify its distributor so as to propagate the update to the watchers. +struct grpc_tls_certificate_provider + : public RefCounted { + public: + grpc_tls_certificate_provider() + : distributor_(MakeRefCounted()), + interested_parties_(grpc_pollset_set_create()) {} + + virtual ~grpc_tls_certificate_provider() { + grpc_pollset_set_destroy(interested_parties_); + } + + grpc_pollset_set* interested_parties() const { return interested_parties_; } + + RefCountedPtr distributor() const { + return distributor_; + } + + private: + RefCountedPtr distributor_; + grpc_pollset_set* interested_parties_; +}; + +} // namespace grpc_core + +#endif // GRPC_CORE_LIB_SECURITY_CERTIFICATE_PROVIDER_H diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 5d601ba7c13..9c3255c7e7d 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -1721,6 +1721,7 @@ src/core/lib/security/authorization/mock_cel/cel_value.h \ src/core/lib/security/authorization/mock_cel/evaluator_core.h \ src/core/lib/security/authorization/mock_cel/flat_expr_builder.h \ src/core/lib/security/authorization/mock_cel/statusor.h \ +src/core/lib/security/certificate_provider.h \ src/core/lib/security/context/security_context.cc \ src/core/lib/security/context/security_context.h \ src/core/lib/security/credentials/alts/alts_credentials.cc \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 3b00ee9aba0..c30a42258e0 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1548,6 +1548,7 @@ src/core/lib/security/authorization/mock_cel/cel_value.h \ src/core/lib/security/authorization/mock_cel/evaluator_core.h \ src/core/lib/security/authorization/mock_cel/flat_expr_builder.h \ src/core/lib/security/authorization/mock_cel/statusor.h \ +src/core/lib/security/certificate_provider.h \ src/core/lib/security/context/security_context.cc \ src/core/lib/security/context/security_context.h \ src/core/lib/security/credentials/alts/alts_credentials.cc \ From f4d18f2d954e472cddb9cfcdc379548cd4bad51c Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Fri, 28 Aug 2020 16:59:55 -0700 Subject: [PATCH 59/62] Reviewer comments --- src/core/lib/security/certificate_provider.h | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/core/lib/security/certificate_provider.h b/src/core/lib/security/certificate_provider.h index 48a37616de0..5a9af3d615e 100644 --- a/src/core/lib/security/certificate_provider.h +++ b/src/core/lib/security/certificate_provider.h @@ -24,8 +24,6 @@ #include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/iomgr/pollset_set.h" -namespace grpc_core { - // TODO(yashkt): After https://github.com/grpc/grpc/pull/23572, remove this // forward declaration and include the header for the distributor instead. struct grpc_tls_certificate_distributor; @@ -43,8 +41,7 @@ struct grpc_tls_certificate_provider : public RefCounted { public: grpc_tls_certificate_provider() - : distributor_(MakeRefCounted()), - interested_parties_(grpc_pollset_set_create()) {} + : interested_parties_(grpc_pollset_set_create()) {} virtual ~grpc_tls_certificate_provider() { grpc_pollset_set_destroy(interested_parties_); @@ -52,15 +49,11 @@ struct grpc_tls_certificate_provider grpc_pollset_set* interested_parties() const { return interested_parties_; } - RefCountedPtr distributor() const { - return distributor_; - } + virtual RefCountedPtr distributor() + const = 0; private: - RefCountedPtr distributor_; grpc_pollset_set* interested_parties_; }; -} // namespace grpc_core - #endif // GRPC_CORE_LIB_SECURITY_CERTIFICATE_PROVIDER_H From 6a5d7106a0d51a87d3250c92e75e71725693fbaf Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Mon, 31 Aug 2020 11:20:44 -0700 Subject: [PATCH 60/62] Pin setuptools to 49.6.0 --- .../internal_ci/helper_scripts/prepare_build_macos_rc | 2 +- tools/internal_ci/macos/grpc_build_artifacts.sh | 10 +++++----- tools/run_tests/artifacts/build_artifact_python.bat | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc index e68a89acc04..875672da4c0 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc @@ -88,7 +88,7 @@ if [ "${PREPARE_BUILD_INSTALL_DEPS_PYTHON}" == "true" ] then # python time pip install --user virtualenv - time pip install --user --upgrade Mako six tox setuptools twisted pyyaml pyjwt cryptography requests + time pip install --user --upgrade Mako six tox setuptools==49.6.0 twisted pyyaml pyjwt cryptography requests export PYTHONPATH=/Library/Python/3.4/site-packages # make sure md5sum is available (requires coreutils 8.31+) diff --git a/tools/internal_ci/macos/grpc_build_artifacts.sh b/tools/internal_ci/macos/grpc_build_artifacts.sh index 10fe02a1215..966737a4d4a 100755 --- a/tools/internal_ci/macos/grpc_build_artifacts.sh +++ b/tools/internal_ci/macos/grpc_build_artifacts.sh @@ -24,11 +24,11 @@ export PREPARE_BUILD_INSTALL_DEPS_RUBY=true source tools/internal_ci/helper_scripts/prepare_build_macos_rc # install cython for all python versions -python2.7 -m pip install -U cython setuptools wheel --user -python3.5 -m pip install -U cython setuptools wheel --user -python3.6 -m pip install -U cython setuptools wheel --user -python3.7 -m pip install -U cython setuptools wheel --user -python3.8 -m pip install -U cython setuptools wheel --user +python2.7 -m pip install -U cython setuptools==49.6.0 wheel --user +python3.5 -m pip install -U cython setuptools==49.6.0 wheel --user +python3.6 -m pip install -U cython setuptools==49.6.0 wheel --user +python3.7 -m pip install -U cython setuptools==49.6.0 wheel --user +python3.8 -m pip install -U cython setuptools==49.6.0 wheel --user # needed to build ruby artifacts time bash tools/distrib/build_ruby_environment_macos.sh diff --git a/tools/run_tests/artifacts/build_artifact_python.bat b/tools/run_tests/artifacts/build_artifact_python.bat index 82d7a48fde6..939853cfe58 100644 --- a/tools/run_tests/artifacts/build_artifact_python.bat +++ b/tools/run_tests/artifacts/build_artifact_python.bat @@ -17,7 +17,7 @@ set PATH=C:\%1;C:\%1\scripts;C:\msys64\mingw%2\bin;C:\tools\msys64\mingw%2\bin;% python -m pip install --upgrade six @rem some artifacts are broken for setuptools 38.5.0. See https://github.com/grpc/grpc/issues/14317 -python -m pip install --upgrade setuptools==38.2.4 +python -m pip install --upgrade setuptools==49.6.0 python -m pip install --upgrade cython python -m pip install -rrequirements.txt --user From 82f0f516904db3fe242284e9c722554359ad3a3b Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Mon, 31 Aug 2020 11:33:03 -0700 Subject: [PATCH 61/62] Pin to 44.1.1 for Python 2 compatibility --- .../internal_ci/helper_scripts/prepare_build_macos_rc | 2 +- tools/internal_ci/macos/grpc_build_artifacts.sh | 10 +++++----- tools/run_tests/artifacts/build_artifact_python.bat | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc index 875672da4c0..e7bc43946c9 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc @@ -88,7 +88,7 @@ if [ "${PREPARE_BUILD_INSTALL_DEPS_PYTHON}" == "true" ] then # python time pip install --user virtualenv - time pip install --user --upgrade Mako six tox setuptools==49.6.0 twisted pyyaml pyjwt cryptography requests + time pip install --user --upgrade Mako six tox setuptools==44.1.1 twisted pyyaml pyjwt cryptography requests export PYTHONPATH=/Library/Python/3.4/site-packages # make sure md5sum is available (requires coreutils 8.31+) diff --git a/tools/internal_ci/macos/grpc_build_artifacts.sh b/tools/internal_ci/macos/grpc_build_artifacts.sh index 966737a4d4a..878d30dc12b 100755 --- a/tools/internal_ci/macos/grpc_build_artifacts.sh +++ b/tools/internal_ci/macos/grpc_build_artifacts.sh @@ -24,11 +24,11 @@ export PREPARE_BUILD_INSTALL_DEPS_RUBY=true source tools/internal_ci/helper_scripts/prepare_build_macos_rc # install cython for all python versions -python2.7 -m pip install -U cython setuptools==49.6.0 wheel --user -python3.5 -m pip install -U cython setuptools==49.6.0 wheel --user -python3.6 -m pip install -U cython setuptools==49.6.0 wheel --user -python3.7 -m pip install -U cython setuptools==49.6.0 wheel --user -python3.8 -m pip install -U cython setuptools==49.6.0 wheel --user +python2.7 -m pip install -U cython setuptools==44.1.1 wheel --user +python3.5 -m pip install -U cython setuptools==44.1.1 wheel --user +python3.6 -m pip install -U cython setuptools==44.1.1 wheel --user +python3.7 -m pip install -U cython setuptools==44.1.1 wheel --user +python3.8 -m pip install -U cython setuptools==44.1.1 wheel --user # needed to build ruby artifacts time bash tools/distrib/build_ruby_environment_macos.sh diff --git a/tools/run_tests/artifacts/build_artifact_python.bat b/tools/run_tests/artifacts/build_artifact_python.bat index 939853cfe58..ff5a6af6fcd 100644 --- a/tools/run_tests/artifacts/build_artifact_python.bat +++ b/tools/run_tests/artifacts/build_artifact_python.bat @@ -17,7 +17,7 @@ set PATH=C:\%1;C:\%1\scripts;C:\msys64\mingw%2\bin;C:\tools\msys64\mingw%2\bin;% python -m pip install --upgrade six @rem some artifacts are broken for setuptools 38.5.0. See https://github.com/grpc/grpc/issues/14317 -python -m pip install --upgrade setuptools==49.6.0 +python -m pip install --upgrade setuptools==44.1.1 python -m pip install --upgrade cython python -m pip install -rrequirements.txt --user From c8fa1c9978c0b4f90fb4a81b0be15a7eafffb8c6 Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Mon, 31 Aug 2020 12:26:24 -0700 Subject: [PATCH 62/62] Pin setuptools in one more location --- tools/run_tests/helper_scripts/build_python.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/run_tests/helper_scripts/build_python.sh b/tools/run_tests/helper_scripts/build_python.sh index 60031fa6e95..a9a186c52bd 100755 --- a/tools/run_tests/helper_scripts/build_python.sh +++ b/tools/run_tests/helper_scripts/build_python.sh @@ -184,8 +184,9 @@ case "$VENV" in ;; esac + +pip_install --upgrade setuptools==44.1.1 pip_install --upgrade pip==19.3.1 -pip_install --upgrade setuptools pip_install --upgrade cython pip_install --upgrade six enum34 protobuf