From 828ee320a4b67c9e1a4b7bf027d0ac7e84082248 Mon Sep 17 00:00:00 2001 From: Taras Galkovskyi Date: Tue, 7 Apr 2020 00:14:41 +0000 Subject: [PATCH 01/47] Expose ALTS client/server credentials on Python layer --- examples/BUILD | 30 +++++ .../python/data_transmission/alts_client.py | 118 ++++++++++++++++++ .../python/data_transmission/alts_server.py | 114 +++++++++++++++++ src/python/grpcio/grpc/__init__.py | 24 ++++ .../grpc/_cython/_cygrpc/credentials.pxd.pxi | 7 +- .../grpc/_cython/_cygrpc/credentials.pyx.pxi | 27 +++- .../grpcio/grpc/_cython/_cygrpc/grpc.pxi | 14 +++ src/python/grpcio_tests/commands.py | 1 + src/python/grpcio_tests/tests/tests.json | 1 + .../grpcio_tests/tests/unit/BUILD.bazel | 1 + .../tests/unit/_alts_credentials_test.py | 56 +++++++++ .../grpcio_tests/tests/unit/_api_test.py | 2 + 12 files changed, 393 insertions(+), 2 deletions(-) create mode 100644 examples/python/data_transmission/alts_client.py create mode 100644 examples/python/data_transmission/alts_server.py create mode 100644 src/python/grpcio_tests/tests/unit/_alts_credentials_test.py diff --git a/examples/BUILD b/examples/BUILD index 5dbde0febe4..a214e4ec977 100644 --- a/examples/BUILD +++ b/examples/BUILD @@ -245,3 +245,33 @@ proto_library( name = "route_guide_proto", srcs = ["protos/route_guide.proto"], ) + +py_binary( + name = "data_transmission_server", + python_version = "PY3", + srcs_version = "PY2AND3", + main = "alts_server.py", + srcs = [ + "python/data_transmission/alts_server.py", + "python/data_transmission/demo_pb2.py", + "python/data_transmission/demo_pb2_grpc.py", + ], + deps = [ + "//src/python/grpcio/grpc:grpcio", + ], +) + +py_binary( + name = "data_transmission_client", + python_version = "PY3", + srcs_version = "PY2AND3", + main = "alts_client.py", + srcs = [ + "python/data_transmission/alts_client.py", + "python/data_transmission/demo_pb2.py", + "python/data_transmission/demo_pb2_grpc.py", + ], + deps = [ + "//src/python/grpcio/grpc:grpcio", + ], +) diff --git a/examples/python/data_transmission/alts_client.py b/examples/python/data_transmission/alts_client.py new file mode 100644 index 00000000000..4595e249b38 --- /dev/null +++ b/examples/python/data_transmission/alts_client.py @@ -0,0 +1,118 @@ +# Copyright 2019 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The example of four ways of data transmission using gRPC in Python.""" + +import time +import grpc + +import demo_pb2_grpc +import demo_pb2 + +SERVER_ADDRESS = "localhost:23333" +CLIENT_ID = 1 + +# 中文注释和英文翻译 +# Note that this example was contributed by an external user using Chinese comments. +# In all cases, the Chinese comment text is translated to English just below it. + + +# 一元模式(在一次调用中, 客户端只能向服务器传输一次请求数据, 服务器也只能返回一次响应) +# unary-unary(In a single call, the client can only send request once, and the server can +# only respond once.) +def simple_method(stub): + print("--------------Call SimpleMethod Begin--------------") + request = demo_pb2.Request(client_id=CLIENT_ID, + request_data="called by Python client") + response = stub.SimpleMethod(request) + print("resp from server(%d), the message=%s" % + (response.server_id, response.response_data)) + print("--------------Call SimpleMethod Over---------------") + + +# 客户端流模式（在一次调用中, 客户端可以多次向服务器传输数据, 但是服务器只能返回一次响应） +# stream-unary (In a single call, the client can transfer data to the server several times, +# but the server can only return a response once.) +def client_streaming_method(stub): + print("--------------Call ClientStreamingMethod Begin--------------") + + # 创建一个生成器 + # create a generator + def request_messages(): + for i in range(5): + request = demo_pb2.Request( + client_id=CLIENT_ID, + request_data=("called by Python client, message:%d" % i)) + yield request + + response = stub.ClientStreamingMethod(request_messages()) + print("resp from server(%d), the message=%s" % + (response.server_id, response.response_data)) + print("--------------Call ClientStreamingMethod Over---------------") + + +# 服务端流模式（在一次调用中, 客户端只能一次向服务器传输数据, 但是服务器可以多次返回响应） +# unary-stream (In a single call, the client can only transmit data to the server at one time, +# but the server can return the response many times.) +def server_streaming_method(stub): + print("--------------Call ServerStreamingMethod Begin--------------") + request = demo_pb2.Request(client_id=CLIENT_ID, + request_data="called by Python client") + response_iterator = stub.ServerStreamingMethod(request) + for response in response_iterator: + print("recv from server(%d), message=%s" % + (response.server_id, response.response_data)) + + print("--------------Call ServerStreamingMethod Over---------------") + + +# 双向流模式 (在一次调用中, 客户端和服务器都可以向对方多次收发数据) +# stream-stream (In a single call, both client and server can send and receive data +# to each other multiple times.) +def bidirectional_streaming_method(stub): + print( + "--------------Call BidirectionalStreamingMethod Begin---------------") + + # 创建一个生成器 + # create a generator + def request_messages(): + for i in range(5): + request = demo_pb2.Request( + client_id=CLIENT_ID, + request_data=("called by Python client, message: %d" % i)) + yield request + time.sleep(1) + + response_iterator = stub.BidirectionalStreamingMethod(request_messages()) + for response in response_iterator: + print("recv from server(%d), message=%s" % + (response.server_id, response.response_data)) + + print("--------------Call BidirectionalStreamingMethod Over---------------") + + +def main(): + with grpc.secure_channel(SERVER_ADDRESS, credentials=grpc.alts_channel_credentials()) as channel: + stub = demo_pb2_grpc.GRPCDemoStub(channel) + + simple_method(stub) + + client_streaming_method(stub) + + server_streaming_method(stub) + + bidirectional_streaming_method(stub) + + +if __name__ == '__main__': + main() diff --git a/examples/python/data_transmission/alts_server.py b/examples/python/data_transmission/alts_server.py new file mode 100644 index 00000000000..05c2dc34ac1 --- /dev/null +++ b/examples/python/data_transmission/alts_server.py @@ -0,0 +1,114 @@ +# Copyright 2019 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The example of four ways of data transmission using gRPC in Python.""" + +from threading import Thread +from concurrent import futures + +import grpc +import demo_pb2_grpc +import demo_pb2 + +SERVER_ADDRESS = 'localhost:23333' +SERVER_ID = 1 + + +class DemoServer(demo_pb2_grpc.GRPCDemoServicer): + + # 一元模式(在一次调用中, 客户端只能向服务器传输一次请求数据, 服务器也只能返回一次响应) + # unary-unary(In a single call, the client can only send request once, and the server can + # only respond once.) + def SimpleMethod(self, request, context): + print("SimpleMethod called by client(%d) the message: %s" % + (request.client_id, request.request_data)) + response = demo_pb2.Response( + server_id=SERVER_ID, + response_data="Python server SimpleMethod Ok!!!!") + return response + + # 客户端流模式（在一次调用中, 客户端可以多次向服务器传输数据, 但是服务器只能返回一次响应） + # stream-unary (In a single call, the client can transfer data to the server several times, + # but the server can only return a response once.) + def ClientStreamingMethod(self, request_iterator, context): + print("ClientStreamingMethod called by client...") + for request in request_iterator: + print("recv from client(%d), message= %s" % + (request.client_id, request.request_data)) + response = demo_pb2.Response( + server_id=SERVER_ID, + response_data="Python server ClientStreamingMethod ok") + return response + + # 服务端流模式（在一次调用中, 客户端只能一次向服务器传输数据, 但是服务器可以多次返回响应） + # unary-stream (In a single call, the client can only transmit data to the server at one time, + # but the server can return the response many times.) + def ServerStreamingMethod(self, request, context): + print("ServerStreamingMethod called by client(%d), message= %s" % + (request.client_id, request.request_data)) + + # 创建一个生成器 + # create a generator + def response_messages(): + for i in range(5): + response = demo_pb2.Response( + server_id=SERVER_ID, + response_data=("send by Python server, message=%d" % i)) + yield response + + return response_messages() + + # 双向流模式 (在一次调用中, 客户端和服务器都可以向对方多次收发数据) + # stream-stream (In a single call, both client and server can send and receive data + # to each other multiple times.) + def BidirectionalStreamingMethod(self, request_iterator, context): + print("BidirectionalStreamingMethod called by client...") + + # 开启一个子线程去接收数据 + # Open a sub thread to receive data + def parse_request(): + for request in request_iterator: + print("recv from client(%d), message= %s" % + (request.client_id, request.request_data)) + + t = Thread(target=parse_request) + t.start() + + for i in range(5): + yield demo_pb2.Response( + server_id=SERVER_ID, + response_data=("send by Python server, message= %d" % i)) + + t.join() + + +def main(): + server = grpc.server(futures.ThreadPoolExecutor()) + + demo_pb2_grpc.add_GRPCDemoServicer_to_server(DemoServer(), server) + + server.add_secure_port(SERVER_ADDRESS, server_credentials=grpc.alts_server_credentials()) + print("------------------start Python GRPC server with ALTS encryption") + server.start() + server.wait_for_termination() + + # If raise Error: + # AttributeError: '_Server' object has no attribute 'wait_for_termination' + # You can use the following code instead: + # import time + # while 1: + # time.sleep(10) + + +if __name__ == '__main__': + main() diff --git a/src/python/grpcio/grpc/__init__.py b/src/python/grpcio/grpc/__init__.py index 9e349c5b023..f3844f39bd3 100644 --- a/src/python/grpcio/grpc/__init__.py +++ b/src/python/grpcio/grpc/__init__.py @@ -1832,6 +1832,28 @@ def local_server_credentials(local_connect_type=LocalConnectionType.LOCAL_TCP): return ServerCredentials( _cygrpc.server_credentials_local(local_connect_type.value)) + +def alts_channel_credentials(): + """Creates a ChannelCredentials for use with an ALTS-enabled Channel. + + This is an EXPERIMENTAL API. + + Returns: + A ChannelCredentials for use with a ALTS-enabled Channel + """ + return ChannelCredentials(_cygrpc.channel_credentials_alts()) + + +def alts_server_credentials(): + """Creates a ServerCredentials for use with an ALTS-enabled connections. + + This is an EXPERIMENTAL API. + + Returns: + A ServerCredentials for use with a local Server + """ + return ServerCredentials(_cygrpc.server_credentials_alts()) + def channel_ready_future(channel): """Creates a Future that tracks when a Channel is ready. @@ -2036,6 +2058,8 @@ __all__ = ( 'composite_channel_credentials', 'local_channel_credentials', 'local_server_credentials', + 'alts_channel_credentials', + 'alts_server_credentials', 'ssl_server_credentials', 'ssl_server_certificate_configuration', 'dynamic_ssl_server_credentials', diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pxd.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pxd.pxi index 0631e1cf63c..2b68117eaa6 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pxd.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pxd.pxi @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - cdef class CallCredentials: cdef grpc_call_credentials *c(self) except * @@ -102,3 +101,9 @@ cdef class ServerCredentials: cdef class LocalChannelCredentials(ChannelCredentials): cdef grpc_local_connect_type _local_connect_type + + +cdef class ALTSChannelCredentials(ChannelCredentials): + cdef grpc_alts_credentials_options *c_options + + cdef grpc_channel_credentials *c(self) except * diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi index 24d1e2a3b77..a45766abc67 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - def _spawn_callback_in_thread(cb_func, args): t = ForkManagedThread(target=cb_func, args=args) t.setDaemon(True) @@ -351,3 +350,29 @@ def server_credentials_local(grpc_local_connect_type local_connect_type): cdef ServerCredentials credentials = ServerCredentials() credentials.c_credentials = grpc_local_server_credentials_create(local_connect_type) return credentials + + +cdef class ALTSChannelCredentials(ChannelCredentials): + + def __cinit__(self): + self.c_options = grpc_alts_credentials_client_options_create() + + def __dealloc__(self): + if self.c_options != NULL: + grpc_alts_credentials_options_destroy(self.c_options) + + cdef grpc_channel_credentials *c(self) except *: + return grpc_alts_credentials_create(self.c_options) + + +def channel_credentials_alts(): + return ALTSChannelCredentials() + + +def server_credentials_alts(): + cdef ServerCredentials credentials = ServerCredentials() + cdef grpc_alts_credentials_options* c_options = grpc_alts_credentials_server_options_create() + credentials.c_credentials = grpc_alts_server_credentials_create(c_options) + # Options can be destroyed as deep copy was performed. + grpc_alts_credentials_options_destroy(c_options) + return credentials diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi index 4a26fdc91d9..e8b164a7004 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi @@ -606,6 +606,20 @@ cdef extern from "grpc/grpc_security.h": grpc_server_credentials *grpc_local_server_credentials_create( grpc_local_connect_type type) + ctypedef struct grpc_alts_credentials_options: + # We don't care about the internals (and in fact don't know them) + pass + + grpc_channel_credentials *grpc_alts_credentials_create( + const grpc_alts_credentials_options *options) + grpc_server_credentials *grpc_alts_server_credentials_create( + const grpc_alts_credentials_options *options) + + grpc_alts_credentials_options* grpc_alts_credentials_client_options_create() + grpc_alts_credentials_options* grpc_alts_credentials_server_options_create() + void grpc_alts_credentials_options_destroy(grpc_alts_credentials_options *options) + + cdef extern from "grpc/compression.h": diff --git a/src/python/grpcio_tests/commands.py b/src/python/grpcio_tests/commands.py index f7cd7c6b8a1..4a7af822b69 100644 --- a/src/python/grpcio_tests/commands.py +++ b/src/python/grpcio_tests/commands.py @@ -220,6 +220,7 @@ class TestGevent(setuptools.Command): 'unit._cython._channel_test.ChannelTest.test_negative_deadline_connectivity', # TODO(https://github.com/grpc/grpc/issues/15411) enable this test 'unit._local_credentials_test.LocalCredentialsTest', + 'unit._alts_credentials_test.ALTSCredentialsTest', 'testing._time_test.StrictRealTimeTest', ) BANNED_WINDOWS_TESTS = ( diff --git a/src/python/grpcio_tests/tests/tests.json b/src/python/grpcio_tests/tests/tests.json index 196e9f08b0a..95fc8c77cf0 100644 --- a/src/python/grpcio_tests/tests/tests.json +++ b/src/python/grpcio_tests/tests/tests.json @@ -24,6 +24,7 @@ "testing._time_test.StrictFakeTimeTest", "testing._time_test.StrictRealTimeTest", "unit._abort_test.AbortTest", + "unit._alts_credentials_test.ALTSCredentialsTest", "unit._api_test.AllTest", "unit._api_test.ChannelConnectivityTest", "unit._api_test.ChannelTest", diff --git a/src/python/grpcio_tests/tests/unit/BUILD.bazel b/src/python/grpcio_tests/tests/unit/BUILD.bazel index 42b99023463..ec2e93538e2 100644 --- a/src/python/grpcio_tests/tests/unit/BUILD.bazel +++ b/src/python/grpcio_tests/tests/unit/BUILD.bazel @@ -4,6 +4,7 @@ package(default_visibility = ["//visibility:public"]) GRPCIO_TESTS_UNIT = [ "_abort_test.py", + "_alts_credentials_test.py", "_api_test.py", "_auth_context_test.py", "_auth_test.py", diff --git a/src/python/grpcio_tests/tests/unit/_alts_credentials_test.py b/src/python/grpcio_tests/tests/unit/_alts_credentials_test.py new file mode 100644 index 00000000000..4a608b80cfa --- /dev/null +++ b/src/python/grpcio_tests/tests/unit/_alts_credentials_test.py @@ -0,0 +1,56 @@ +# Copyright 2020 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Test of RPCs made using ALTS credentials.""" + +import unittest +import os +from concurrent.futures import ThreadPoolExecutor +import grpc + + +class _GenericHandler(grpc.GenericRpcHandler): + + def service(self, handler_call_details): + return grpc.unary_unary_rpc_method_handler( + lambda request, unused_context: request) + + +class ALTSCredentialsTest(unittest.TestCase): + + def _create_server(self): + server = grpc.server(ThreadPoolExecutor()) + server.add_generic_rpc_handlers((_GenericHandler(),)) + return server + + @unittest.skipIf(os.name == 'nt', + 'TODO(https://github.com/grpc/grpc/issues/20078)') + def test_alts(self): + server_addr = 'localhost:{}' + channel_creds = grpc.alts_channel_credentials() + server_creds = grpc.alts_server_credentials() + + server = self._create_server() + port = server.add_secure_port(server_addr.format(0), server_creds) + server.start() + with grpc.secure_channel(server_addr.format(port), + channel_creds) as channel: + self.assertEqual( + b'abc', + channel.unary_unary('/test/method')(b'abc', + wait_for_ready=True)) + server.stop(None) + + +if __name__ == '__main__': + unittest.main() diff --git a/src/python/grpcio_tests/tests/unit/_api_test.py b/src/python/grpcio_tests/tests/unit/_api_test.py index cc0daba2098..a37ac3ef319 100644 --- a/src/python/grpcio_tests/tests/unit/_api_test.py +++ b/src/python/grpcio_tests/tests/unit/_api_test.py @@ -63,6 +63,8 @@ class AllTest(unittest.TestCase): 'LocalConnectionType', 'local_channel_credentials', 'local_server_credentials', + 'alts_channel_credentials', + 'alts_server_credentials', 'unary_unary_rpc_method_handler', 'unary_stream_rpc_method_handler', 'stream_unary_rpc_method_handler', From d84d53f90150c4d783275c691cf0add7d2b12ad1 Mon Sep 17 00:00:00 2001 From: Taras Galkovskyi Date: Tue, 7 Apr 2020 00:26:23 +0000 Subject: [PATCH 02/47] fix comment --- src/python/grpcio/grpc/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/python/grpcio/grpc/__init__.py b/src/python/grpcio/grpc/__init__.py index f3844f39bd3..04c3f69a475 100644 --- a/src/python/grpcio/grpc/__init__.py +++ b/src/python/grpcio/grpc/__init__.py @@ -1839,7 +1839,7 @@ def alts_channel_credentials(): This is an EXPERIMENTAL API. Returns: - A ChannelCredentials for use with a ALTS-enabled Channel + A ChannelCredentials for use with an ALTS-enabled Channel """ return ChannelCredentials(_cygrpc.channel_credentials_alts()) @@ -1850,7 +1850,7 @@ def alts_server_credentials(): This is an EXPERIMENTAL API. Returns: - A ServerCredentials for use with a local Server + A ServerCredentials for use with an ALTS-enabled Server """ return ServerCredentials(_cygrpc.server_credentials_alts()) From 309c8e65f4dc9fcf2a8a355f8dfed03d1061a0cf Mon Sep 17 00:00:00 2001 From: Taras Galkovskyi Date: Thu, 9 Apr 2020 21:39:08 +0000 Subject: [PATCH 03/47] declare alts support for python interop client/server --- .../grpcio_tests/tests/interop/client.py | 39 +++++++++++-------- .../grpcio_tests/tests/interop/server.py | 11 ++++++ .../grpcio_tests/tests_aio/interop/client.py | 2 +- .../grpcio_tests/tests_aio/interop/server.py | 8 ++-- tools/run_tests/run_interop_tests.py | 4 +- 5 files changed, 42 insertions(+), 22 deletions(-) diff --git a/src/python/grpcio_tests/tests/interop/client.py b/src/python/grpcio_tests/tests/interop/client.py index 71b2fecef18..6537413dbdd 100644 --- a/src/python/grpcio_tests/tests/interop/client.py +++ b/src/python/grpcio_tests/tests/interop/client.py @@ -43,6 +43,10 @@ def parse_interop_client_args(): default=False, type=resources.parse_bool, help='require a secure connection') + parser.add_argument('--use_alts', + default=False, + type=resources.parse_bool, + help='require an ALTS secure connection') parser.add_argument('--use_test_ca', default=False, type=resources.parse_bool, @@ -85,22 +89,25 @@ def _create_call_credentials(args): def get_secure_channel_parameters(args): call_credentials = _create_call_credentials(args) - if args.use_test_ca: - root_certificates = resources.test_root_certificates() - else: - root_certificates = None # will load default roots. - - channel_credentials = grpc.ssl_channel_credentials(root_certificates) - if call_credentials is not None: - channel_credentials = grpc.composite_channel_credentials( - channel_credentials, call_credentials) - channel_opts = None - if args.server_host_override: - channel_opts = (( - 'grpc.ssl_target_name_override', - args.server_host_override, - ),) + if args.use_tls: + if args.use_test_ca: + root_certificates = resources.test_root_certificates() + else: + root_certificates = None # will load default roots. + + channel_credentials = grpc.ssl_channel_credentials(root_certificates) + if call_credentials is not None: + channel_credentials = grpc.composite_channel_credentials( + channel_credentials, call_credentials) + + if args.server_host_override: + channel_opts = (( + 'grpc.ssl_target_name_override', + args.server_host_override, + ),) + else args.use_alts: + channel_credentials = grpc.alts_channel_credentials() return channel_credentials, channel_opts @@ -108,7 +115,7 @@ def get_secure_channel_parameters(args): def _create_channel(args): target = '{}:{}'.format(args.server_host, args.server_port) - if args.use_tls: + if args.use_tls or args.use_alts: channel_credentials, options = get_secure_channel_parameters(args) return grpc.secure_channel(target, channel_credentials, options) else: diff --git a/src/python/grpcio_tests/tests/interop/server.py b/src/python/grpcio_tests/tests/interop/server.py index 0a0061f5b1c..852b13c7b94 100644 --- a/src/python/grpcio_tests/tests/interop/server.py +++ b/src/python/grpcio_tests/tests/interop/server.py @@ -38,6 +38,10 @@ def parse_interop_server_arguments(): default=False, type=resources.parse_bool, help='require a secure connection') + parser.add_argument('--use_alts', + default=False, + type=resources.parse_bool, + help='require an ALTS connection') return parser.parse_args() @@ -47,6 +51,10 @@ def get_server_credentials(): return grpc.ssl_server_credentials(((private_key, certificate_chain),)) +def get_alts_server_credentials(): + return grpc.alts_server_credentials() + + def serve(): args = parse_interop_server_arguments() @@ -56,6 +64,9 @@ def serve(): if args.use_tls: credentials = get_server_credentials() server.add_secure_port('[::]:{}'.format(args.port), credentials) + elif args.use_alts: + credentials = get_alts_server_credentials() + server.add_secure_port('[::]:{}'.format(args.port), credentials) else: server.add_insecure_port('[::]:{}'.format(args.port)) diff --git a/src/python/grpcio_tests/tests_aio/interop/client.py b/src/python/grpcio_tests/tests_aio/interop/client.py index 89793f94054..7731f3bd6eb 100644 --- a/src/python/grpcio_tests/tests_aio/interop/client.py +++ b/src/python/grpcio_tests/tests_aio/interop/client.py @@ -30,7 +30,7 @@ _LOGGER.setLevel(logging.DEBUG) def _create_channel(args): target = f'{args.server_host}:{args.server_port}' - if args.use_tls: + if args.use_tls or args.use_alts: channel_credentials, options = interop_client_lib.get_secure_channel_parameters( args) return aio.secure_channel(target, channel_credentials, options) diff --git a/src/python/grpcio_tests/tests_aio/interop/server.py b/src/python/grpcio_tests/tests_aio/interop/server.py index 06a6c51d13a..ae5ee95769e 100644 --- a/src/python/grpcio_tests/tests_aio/interop/server.py +++ b/src/python/grpcio_tests/tests_aio/interop/server.py @@ -30,9 +30,11 @@ _LOGGER.setLevel(logging.DEBUG) async def serve(): args = interop_server_lib.parse_interop_server_arguments() - if args.use_tls: - credentials = interop_server_lib.get_server_credentials() - + if args.use_tls or args.use_alts: + if args.use_tls: + credentials = interop_server_lib.get_server_credentials() + else: + credentials = interop_server_lib.get_alts_server_credentials() address, server = await _test_server.start_test_server( port=args.port, secure=True, server_credentials=credentials) else: diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index d183c5b4b47..63920f78f31 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -786,9 +786,9 @@ _LANGUAGES_WITH_HTTP2_CLIENTS_FOR_HTTP2_SERVER_TEST_CASES = [ 'java', 'go', 'python', 'c++' ] -_LANGUAGES_FOR_ALTS_TEST_CASES = ['java', 'go', 'c++'] +_LANGUAGES_FOR_ALTS_TEST_CASES = ['java', 'go', 'c++', 'python'] -_SERVERS_FOR_ALTS_TEST_CASES = ['java', 'go', 'c++'] +_SERVERS_FOR_ALTS_TEST_CASES = ['java', 'go', 'c++', 'python'] _TRANSPORT_SECURITY_OPTIONS = ['tls', 'alts', 'insecure'] From ed04792ac54865ebe13ed57d9d66dacaca240dd6 Mon Sep 17 00:00:00 2001 From: Taras Galkovskyi Date: Thu, 9 Apr 2020 22:36:58 +0000 Subject: [PATCH 04/47] unify interop alts credentials use --- src/python/grpcio_tests/tests/interop/server.py | 16 +++++++--------- .../grpcio_tests/tests_aio/interop/server.py | 5 +---- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/src/python/grpcio_tests/tests/interop/server.py b/src/python/grpcio_tests/tests/interop/server.py index 852b13c7b94..e389533aa85 100644 --- a/src/python/grpcio_tests/tests/interop/server.py +++ b/src/python/grpcio_tests/tests/interop/server.py @@ -46,13 +46,14 @@ def parse_interop_server_arguments(): def get_server_credentials(): - private_key = resources.private_key() - certificate_chain = resources.certificate_chain() - return grpc.ssl_server_credentials(((private_key, certificate_chain),)) + if args.use_tls: + private_key = resources.private_key() + certificate_chain = resources.certificate_chain() + return grpc.ssl_server_credentials(((private_key, certificate_chain),)) + else: + return grpc.alts_server_credentials() -def get_alts_server_credentials(): - return grpc.alts_server_credentials() def serve(): @@ -61,12 +62,9 @@ def serve(): server = test_common.test_server() test_pb2_grpc.add_TestServiceServicer_to_server(service.TestService(), server) - if args.use_tls: + if args.use_tls or args.use_alts: credentials = get_server_credentials() server.add_secure_port('[::]:{}'.format(args.port), credentials) - elif args.use_alts: - credentials = get_alts_server_credentials() - server.add_secure_port('[::]:{}'.format(args.port), credentials) else: server.add_insecure_port('[::]:{}'.format(args.port)) diff --git a/src/python/grpcio_tests/tests_aio/interop/server.py b/src/python/grpcio_tests/tests_aio/interop/server.py index ae5ee95769e..82c52cebf0c 100644 --- a/src/python/grpcio_tests/tests_aio/interop/server.py +++ b/src/python/grpcio_tests/tests_aio/interop/server.py @@ -31,10 +31,7 @@ async def serve(): args = interop_server_lib.parse_interop_server_arguments() if args.use_tls or args.use_alts: - if args.use_tls: - credentials = interop_server_lib.get_server_credentials() - else: - credentials = interop_server_lib.get_alts_server_credentials() + credentials = interop_server_lib.get_server_credentials() address, server = await _test_server.start_test_server( port=args.port, secure=True, server_credentials=credentials) else: From 2b3eca0858f26cac5f2ecf47d1c3172c4fdb7fef Mon Sep 17 00:00:00 2001 From: Taras Galkovskyi Date: Fri, 10 Apr 2020 14:09:17 +0000 Subject: [PATCH 05/47] adding api to configure target service accounts for client ALTS --- src/python/grpcio/grpc/__init__.py | 8 +++++--- .../grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi | 8 +++++--- src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi | 1 + src/python/grpcio_tests/tests/interop/client.py | 2 +- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/python/grpcio/grpc/__init__.py b/src/python/grpcio/grpc/__init__.py index 04c3f69a475..ac17c53d6e9 100644 --- a/src/python/grpcio/grpc/__init__.py +++ b/src/python/grpcio/grpc/__init__.py @@ -1833,22 +1833,24 @@ def local_server_credentials(local_connect_type=LocalConnectionType.LOCAL_TCP): _cygrpc.server_credentials_local(local_connect_type.value)) -def alts_channel_credentials(): +def alts_channel_credentials(service_accounts=[]): """Creates a ChannelCredentials for use with an ALTS-enabled Channel. This is an EXPERIMENTAL API. + Args: + service_accounts: list of strings, target service accounts Returns: A ChannelCredentials for use with an ALTS-enabled Channel """ - return ChannelCredentials(_cygrpc.channel_credentials_alts()) + return ChannelCredentials(_cygrpc.channel_credentials_alts(service_accounts)) def alts_server_credentials(): """Creates a ServerCredentials for use with an ALTS-enabled connections. This is an EXPERIMENTAL API. - + Returns: A ServerCredentials for use with an ALTS-enabled Server """ diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi index a45766abc67..3cc11362bc6 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi @@ -354,8 +354,10 @@ def server_credentials_local(grpc_local_connect_type local_connect_type): cdef class ALTSChannelCredentials(ChannelCredentials): - def __cinit__(self): + def __cinit__(self, service_accounts): self.c_options = grpc_alts_credentials_client_options_create() + for account in service_accounts: + grpc_alts_credentials_client_options_add_target_service_account(self.c_options, account) def __dealloc__(self): if self.c_options != NULL: @@ -365,8 +367,8 @@ cdef class ALTSChannelCredentials(ChannelCredentials): return grpc_alts_credentials_create(self.c_options) -def channel_credentials_alts(): - return ALTSChannelCredentials() +def channel_credentials_alts(service_accounts): + return ALTSChannelCredentials(service_accounts) def server_credentials_alts(): diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi index e8b164a7004..98a71f92699 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi @@ -618,6 +618,7 @@ cdef extern from "grpc/grpc_security.h": grpc_alts_credentials_options* grpc_alts_credentials_client_options_create() grpc_alts_credentials_options* grpc_alts_credentials_server_options_create() void grpc_alts_credentials_options_destroy(grpc_alts_credentials_options *options) + void grpc_alts_credentials_client_options_add_target_service_account(grpc_alts_credentials_options *options, const char *service_account) diff --git a/src/python/grpcio_tests/tests/interop/client.py b/src/python/grpcio_tests/tests/interop/client.py index 6537413dbdd..03f6113a88c 100644 --- a/src/python/grpcio_tests/tests/interop/client.py +++ b/src/python/grpcio_tests/tests/interop/client.py @@ -107,7 +107,7 @@ def get_secure_channel_parameters(args): args.server_host_override, ),) else args.use_alts: - channel_credentials = grpc.alts_channel_credentials() + channel_credentials = grpc.alts_channel_credentials(['svc_account1@gmail.com']) return channel_credentials, channel_opts From 83ada3b598671946c482c955445e6ec44bdb5e95 Mon Sep 17 00:00:00 2001 From: Taras Galkovskyi Date: Fri, 10 Apr 2020 14:20:27 +0000 Subject: [PATCH 06/47] adjusting formatting & tests --- src/python/grpcio/grpc/_cython/_cygrpc/credentials.pxd.pxi | 1 + src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi | 1 + src/python/grpcio_tests/tests/unit/_alts_credentials_test.py | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pxd.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pxd.pxi index 2b68117eaa6..ddaedb30bd4 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pxd.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pxd.pxi @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + cdef class CallCredentials: cdef grpc_call_credentials *c(self) except * diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi index 3cc11362bc6..e7ed15cd16b 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + def _spawn_callback_in_thread(cb_func, args): t = ForkManagedThread(target=cb_func, args=args) t.setDaemon(True) diff --git a/src/python/grpcio_tests/tests/unit/_alts_credentials_test.py b/src/python/grpcio_tests/tests/unit/_alts_credentials_test.py index 4a608b80cfa..a6809c02a1a 100644 --- a/src/python/grpcio_tests/tests/unit/_alts_credentials_test.py +++ b/src/python/grpcio_tests/tests/unit/_alts_credentials_test.py @@ -37,7 +37,7 @@ class ALTSCredentialsTest(unittest.TestCase): 'TODO(https://github.com/grpc/grpc/issues/20078)') def test_alts(self): server_addr = 'localhost:{}' - channel_creds = grpc.alts_channel_credentials() + channel_creds = grpc.alts_channel_credentials(['svcacct@server.com']) server_creds = grpc.alts_server_credentials() server = self._create_server() From 72cd29266d29df73a249fc406a72acdb3be4dc88 Mon Sep 17 00:00:00 2001 From: Taras Galkovskyi Date: Fri, 10 Apr 2020 14:27:30 +0000 Subject: [PATCH 07/47] simplify ALTS client --- .../python/data_transmission/alts_client.py | 97 ++----------------- .../python/data_transmission/alts_server.py | 91 ++--------------- 2 files changed, 14 insertions(+), 174 deletions(-) diff --git a/examples/python/data_transmission/alts_client.py b/examples/python/data_transmission/alts_client.py index 4595e249b38..a1dc8ec558e 100644 --- a/examples/python/data_transmission/alts_client.py +++ b/examples/python/data_transmission/alts_client.py @@ -1,4 +1,4 @@ -# Copyright 2019 gRPC authors. +# Copyright 2020 gRPC authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,107 +11,24 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""The example of four ways of data transmission using gRPC in Python.""" +"""The example of using ALTS credentials to setup gRPC client.""" import time import grpc +import client import demo_pb2_grpc import demo_pb2 SERVER_ADDRESS = "localhost:23333" -CLIENT_ID = 1 - -# 中文注释和英文翻译 -# Note that this example was contributed by an external user using Chinese comments. -# In all cases, the Chinese comment text is translated to English just below it. - - -# 一元模式(在一次调用中, 客户端只能向服务器传输一次请求数据, 服务器也只能返回一次响应) -# unary-unary(In a single call, the client can only send request once, and the server can -# only respond once.) -def simple_method(stub): - print("--------------Call SimpleMethod Begin--------------") - request = demo_pb2.Request(client_id=CLIENT_ID, - request_data="called by Python client") - response = stub.SimpleMethod(request) - print("resp from server(%d), the message=%s" % - (response.server_id, response.response_data)) - print("--------------Call SimpleMethod Over---------------") - - -# 客户端流模式（在一次调用中, 客户端可以多次向服务器传输数据, 但是服务器只能返回一次响应） -# stream-unary (In a single call, the client can transfer data to the server several times, -# but the server can only return a response once.) -def client_streaming_method(stub): - print("--------------Call ClientStreamingMethod Begin--------------") - - # 创建一个生成器 - # create a generator - def request_messages(): - for i in range(5): - request = demo_pb2.Request( - client_id=CLIENT_ID, - request_data=("called by Python client, message:%d" % i)) - yield request - - response = stub.ClientStreamingMethod(request_messages()) - print("resp from server(%d), the message=%s" % - (response.server_id, response.response_data)) - print("--------------Call ClientStreamingMethod Over---------------") - - -# 服务端流模式（在一次调用中, 客户端只能一次向服务器传输数据, 但是服务器可以多次返回响应） -# unary-stream (In a single call, the client can only transmit data to the server at one time, -# but the server can return the response many times.) -def server_streaming_method(stub): - print("--------------Call ServerStreamingMethod Begin--------------") - request = demo_pb2.Request(client_id=CLIENT_ID, - request_data="called by Python client") - response_iterator = stub.ServerStreamingMethod(request) - for response in response_iterator: - print("recv from server(%d), message=%s" % - (response.server_id, response.response_data)) - - print("--------------Call ServerStreamingMethod Over---------------") - - -# 双向流模式 (在一次调用中, 客户端和服务器都可以向对方多次收发数据) -# stream-stream (In a single call, both client and server can send and receive data -# to each other multiple times.) -def bidirectional_streaming_method(stub): - print( - "--------------Call BidirectionalStreamingMethod Begin---------------") - - # 创建一个生成器 - # create a generator - def request_messages(): - for i in range(5): - request = demo_pb2.Request( - client_id=CLIENT_ID, - request_data=("called by Python client, message: %d" % i)) - yield request - time.sleep(1) - - response_iterator = stub.BidirectionalStreamingMethod(request_messages()) - for response in response_iterator: - print("recv from server(%d), message=%s" % - (response.server_id, response.response_data)) - - print("--------------Call BidirectionalStreamingMethod Over---------------") - def main(): with grpc.secure_channel(SERVER_ADDRESS, credentials=grpc.alts_channel_credentials()) as channel: stub = demo_pb2_grpc.GRPCDemoStub(channel) - - simple_method(stub) - - client_streaming_method(stub) - - server_streaming_method(stub) - - bidirectional_streaming_method(stub) + client.simple_method(stub) + client.client_streaming_method(stub) + client.server_streaming_method(stub) + client.bidirectional_streaming_method(stub) if __name__ == '__main__': diff --git a/examples/python/data_transmission/alts_server.py b/examples/python/data_transmission/alts_server.py index 05c2dc34ac1..29a8417152e 100644 --- a/examples/python/data_transmission/alts_server.py +++ b/examples/python/data_transmission/alts_server.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""The example of four ways of data transmission using gRPC in Python.""" +"""The example of using ALTS credentials to setup gRPC server in python.""" from threading import Thread from concurrent import futures @@ -19,95 +19,18 @@ from concurrent import futures import grpc import demo_pb2_grpc import demo_pb2 +import server SERVER_ADDRESS = 'localhost:23333' -SERVER_ID = 1 - - -class DemoServer(demo_pb2_grpc.GRPCDemoServicer): - - # 一元模式(在一次调用中, 客户端只能向服务器传输一次请求数据, 服务器也只能返回一次响应) - # unary-unary(In a single call, the client can only send request once, and the server can - # only respond once.) - def SimpleMethod(self, request, context): - print("SimpleMethod called by client(%d) the message: %s" % - (request.client_id, request.request_data)) - response = demo_pb2.Response( - server_id=SERVER_ID, - response_data="Python server SimpleMethod Ok!!!!") - return response - - # 客户端流模式（在一次调用中, 客户端可以多次向服务器传输数据, 但是服务器只能返回一次响应） - # stream-unary (In a single call, the client can transfer data to the server several times, - # but the server can only return a response once.) - def ClientStreamingMethod(self, request_iterator, context): - print("ClientStreamingMethod called by client...") - for request in request_iterator: - print("recv from client(%d), message= %s" % - (request.client_id, request.request_data)) - response = demo_pb2.Response( - server_id=SERVER_ID, - response_data="Python server ClientStreamingMethod ok") - return response - - # 服务端流模式（在一次调用中, 客户端只能一次向服务器传输数据, 但是服务器可以多次返回响应） - # unary-stream (In a single call, the client can only transmit data to the server at one time, - # but the server can return the response many times.) - def ServerStreamingMethod(self, request, context): - print("ServerStreamingMethod called by client(%d), message= %s" % - (request.client_id, request.request_data)) - - # 创建一个生成器 - # create a generator - def response_messages(): - for i in range(5): - response = demo_pb2.Response( - server_id=SERVER_ID, - response_data=("send by Python server, message=%d" % i)) - yield response - - return response_messages() - - # 双向流模式 (在一次调用中, 客户端和服务器都可以向对方多次收发数据) - # stream-stream (In a single call, both client and server can send and receive data - # to each other multiple times.) - def BidirectionalStreamingMethod(self, request_iterator, context): - print("BidirectionalStreamingMethod called by client...") - - # 开启一个子线程去接收数据 - # Open a sub thread to receive data - def parse_request(): - for request in request_iterator: - print("recv from client(%d), message= %s" % - (request.client_id, request.request_data)) - - t = Thread(target=parse_request) - t.start() - - for i in range(5): - yield demo_pb2.Response( - server_id=SERVER_ID, - response_data=("send by Python server, message= %d" % i)) - - t.join() def main(): - server = grpc.server(futures.ThreadPoolExecutor()) - - demo_pb2_grpc.add_GRPCDemoServicer_to_server(DemoServer(), server) - - server.add_secure_port(SERVER_ADDRESS, server_credentials=grpc.alts_server_credentials()) + svr = grpc.server(futures.ThreadPoolExecutor()) + demo_pb2_grpc.add_GRPCDemoServicer_to_server(server.DemoServer(), svr) + svr.add_secure_port(SERVER_ADDRESS, server_credentials=grpc.alts_server_credentials()) print("------------------start Python GRPC server with ALTS encryption") - server.start() - server.wait_for_termination() - - # If raise Error: - # AttributeError: '_Server' object has no attribute 'wait_for_termination' - # You can use the following code instead: - # import time - # while 1: - # time.sleep(10) + svr.start() + svr.wait_for_termination() if __name__ == '__main__': From 97e7c5e9b6dbb34439f355a77a2e66d42593e41e Mon Sep 17 00:00:00 2001 From: Taras Galkovskyi Date: Fri, 10 Apr 2020 14:30:57 +0000 Subject: [PATCH 08/47] ALTS BUILD file for exampel --- examples/BUILD | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/BUILD b/examples/BUILD index a214e4ec977..057940abd3a 100644 --- a/examples/BUILD +++ b/examples/BUILD @@ -255,6 +255,7 @@ py_binary( "python/data_transmission/alts_server.py", "python/data_transmission/demo_pb2.py", "python/data_transmission/demo_pb2_grpc.py", + "python/data_transmission/server.py", ], deps = [ "//src/python/grpcio/grpc:grpcio", @@ -268,6 +269,7 @@ py_binary( main = "alts_client.py", srcs = [ "python/data_transmission/alts_client.py", + "python/data_transmission/client.py", "python/data_transmission/demo_pb2.py", "python/data_transmission/demo_pb2_grpc.py", ], From 02da3330c26a706f2de820334c88abe3eea7e725 Mon Sep 17 00:00:00 2001 From: Taras Galkovskyi Date: Wed, 15 Apr 2020 18:13:04 +0000 Subject: [PATCH 09/47] address review comments --- examples/BUILD | 32 ------------ examples/python/data_transmission/BUILD | 49 +++++++++++++++++++ .../python/data_transmission/alts_client.py | 2 - .../python/data_transmission/alts_server.py | 2 - .../grpc/_cython/_cygrpc/credentials.pyx.pxi | 4 +- src/python/grpcio_tests/commands.py | 1 - .../grpcio_tests/tests/interop/client.py | 4 +- .../tests/unit/_alts_credentials_test.py | 11 +++-- 8 files changed, 59 insertions(+), 46 deletions(-) create mode 100644 examples/python/data_transmission/BUILD diff --git a/examples/BUILD b/examples/BUILD index 057940abd3a..5dbde0febe4 100644 --- a/examples/BUILD +++ b/examples/BUILD @@ -245,35 +245,3 @@ proto_library( name = "route_guide_proto", srcs = ["protos/route_guide.proto"], ) - -py_binary( - name = "data_transmission_server", - python_version = "PY3", - srcs_version = "PY2AND3", - main = "alts_server.py", - srcs = [ - "python/data_transmission/alts_server.py", - "python/data_transmission/demo_pb2.py", - "python/data_transmission/demo_pb2_grpc.py", - "python/data_transmission/server.py", - ], - deps = [ - "//src/python/grpcio/grpc:grpcio", - ], -) - -py_binary( - name = "data_transmission_client", - python_version = "PY3", - srcs_version = "PY2AND3", - main = "alts_client.py", - srcs = [ - "python/data_transmission/alts_client.py", - "python/data_transmission/client.py", - "python/data_transmission/demo_pb2.py", - "python/data_transmission/demo_pb2_grpc.py", - ], - deps = [ - "//src/python/grpcio/grpc:grpcio", - ], -) diff --git a/examples/python/data_transmission/BUILD b/examples/python/data_transmission/BUILD new file mode 100644 index 00000000000..528cd325a1b --- /dev/null +++ b/examples/python/data_transmission/BUILD @@ -0,0 +1,49 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +licenses(["notice"]) # 3-clause BSD + +load("@grpc_python_dependencies//:requirements.bzl", "requirement") + +py_binary( + name = "alts_server", + python_version = "PY3", + srcs_version = "PY2AND3", + main = "alts_server.py", + srcs = [ + "alts_server.py", + "demo_pb2.py", + "demo_pb2_grpc.py", + "server.py", + ], + deps = [ + "//src/python/grpcio/grpc:grpcio", + ], +) + +py_binary( + name = "alts_client", + python_version = "PY3", + srcs_version = "PY2AND3", + main = "alts_client.py", + srcs = [ + "alts_client.py", + "client.py", + "demo_pb2.py", + "demo_pb2_grpc.py", + ], + deps = [ + "//src/python/grpcio/grpc:grpcio", + ], +) diff --git a/examples/python/data_transmission/alts_client.py b/examples/python/data_transmission/alts_client.py index a1dc8ec558e..e3ddb281930 100644 --- a/examples/python/data_transmission/alts_client.py +++ b/examples/python/data_transmission/alts_client.py @@ -13,12 +13,10 @@ # limitations under the License. """The example of using ALTS credentials to setup gRPC client.""" -import time import grpc import client import demo_pb2_grpc -import demo_pb2 SERVER_ADDRESS = "localhost:23333" diff --git a/examples/python/data_transmission/alts_server.py b/examples/python/data_transmission/alts_server.py index 29a8417152e..e91a1545377 100644 --- a/examples/python/data_transmission/alts_server.py +++ b/examples/python/data_transmission/alts_server.py @@ -13,12 +13,10 @@ # limitations under the License. """The example of using ALTS credentials to setup gRPC server in python.""" -from threading import Thread from concurrent import futures import grpc import demo_pb2_grpc -import demo_pb2 import server SERVER_ADDRESS = 'localhost:23333' diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi index e7ed15cd16b..d3d0f758b16 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi @@ -355,7 +355,7 @@ def server_credentials_local(grpc_local_connect_type local_connect_type): cdef class ALTSChannelCredentials(ChannelCredentials): - def __cinit__(self, service_accounts): + def __cinit__(self, list service_accounts): self.c_options = grpc_alts_credentials_client_options_create() for account in service_accounts: grpc_alts_credentials_client_options_add_target_service_account(self.c_options, account) @@ -368,7 +368,7 @@ cdef class ALTSChannelCredentials(ChannelCredentials): return grpc_alts_credentials_create(self.c_options) -def channel_credentials_alts(service_accounts): +def channel_credentials_alts(list service_accounts): return ALTSChannelCredentials(service_accounts) diff --git a/src/python/grpcio_tests/commands.py b/src/python/grpcio_tests/commands.py index 4a7af822b69..f7cd7c6b8a1 100644 --- a/src/python/grpcio_tests/commands.py +++ b/src/python/grpcio_tests/commands.py @@ -220,7 +220,6 @@ class TestGevent(setuptools.Command): 'unit._cython._channel_test.ChannelTest.test_negative_deadline_connectivity', # TODO(https://github.com/grpc/grpc/issues/15411) enable this test 'unit._local_credentials_test.LocalCredentialsTest', - 'unit._alts_credentials_test.ALTSCredentialsTest', 'testing._time_test.StrictRealTimeTest', ) BANNED_WINDOWS_TESTS = ( diff --git a/src/python/grpcio_tests/tests/interop/client.py b/src/python/grpcio_tests/tests/interop/client.py index 03f6113a88c..04b82c0b401 100644 --- a/src/python/grpcio_tests/tests/interop/client.py +++ b/src/python/grpcio_tests/tests/interop/client.py @@ -106,8 +106,8 @@ def get_secure_channel_parameters(args): 'grpc.ssl_target_name_override', args.server_host_override, ),) - else args.use_alts: - channel_credentials = grpc.alts_channel_credentials(['svc_account1@gmail.com']) + elif args.use_alts: + channel_credentials = grpc.alts_channel_credentials() return channel_credentials, channel_opts diff --git a/src/python/grpcio_tests/tests/unit/_alts_credentials_test.py b/src/python/grpcio_tests/tests/unit/_alts_credentials_test.py index a6809c02a1a..07a3c849c8b 100644 --- a/src/python/grpcio_tests/tests/unit/_alts_credentials_test.py +++ b/src/python/grpcio_tests/tests/unit/_alts_credentials_test.py @@ -19,6 +19,9 @@ from concurrent.futures import ThreadPoolExecutor import grpc +REQUEST = b'abc' + + class _GenericHandler(grpc.GenericRpcHandler): def service(self, handler_call_details): @@ -33,11 +36,9 @@ class ALTSCredentialsTest(unittest.TestCase): server.add_generic_rpc_handlers((_GenericHandler(),)) return server - @unittest.skipIf(os.name == 'nt', - 'TODO(https://github.com/grpc/grpc/issues/20078)') def test_alts(self): server_addr = 'localhost:{}' - channel_creds = grpc.alts_channel_credentials(['svcacct@server.com']) + channel_creds = grpc.alts_channel_credentials([]) server_creds = grpc.alts_server_credentials() server = self._create_server() @@ -46,8 +47,8 @@ class ALTSCredentialsTest(unittest.TestCase): with grpc.secure_channel(server_addr.format(port), channel_creds) as channel: self.assertEqual( - b'abc', - channel.unary_unary('/test/method')(b'abc', + REQUEST, + channel.unary_unary('/test/method')(REQUEST, wait_for_ready=True)) server.stop(None) From 42bd1068980b7948620620c1e3985f8005a01ab8 Mon Sep 17 00:00:00 2001 From: Taras Galkovskyi Date: Wed, 15 Apr 2020 22:14:08 +0000 Subject: [PATCH 10/47] updating comments to make clear that ALTS is only supported on GCP --- examples/python/data_transmission/alts_client.py | 4 +++- examples/python/data_transmission/alts_server.py | 4 +++- src/python/grpcio/grpc/__init__.py | 14 ++++++++++++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/examples/python/data_transmission/alts_client.py b/examples/python/data_transmission/alts_client.py index e3ddb281930..0bfcede9465 100644 --- a/examples/python/data_transmission/alts_client.py +++ b/examples/python/data_transmission/alts_client.py @@ -11,7 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""The example of using ALTS credentials to setup gRPC client.""" +"""The example of using ALTS credentials to setup gRPC client. + +The example would only successfully run in GCP environment.""" import grpc diff --git a/examples/python/data_transmission/alts_server.py b/examples/python/data_transmission/alts_server.py index e91a1545377..4f7b84d9ca6 100644 --- a/examples/python/data_transmission/alts_server.py +++ b/examples/python/data_transmission/alts_server.py @@ -11,7 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""The example of using ALTS credentials to setup gRPC server in python.""" +"""The example of using ALTS credentials to setup gRPC server in python. + +The example would only successfully run in GCP environment.""" from concurrent import futures diff --git a/src/python/grpcio/grpc/__init__.py b/src/python/grpcio/grpc/__init__.py index ac17c53d6e9..7d5d85f26ef 100644 --- a/src/python/grpcio/grpc/__init__.py +++ b/src/python/grpcio/grpc/__init__.py @@ -1837,9 +1837,16 @@ def alts_channel_credentials(service_accounts=[]): """Creates a ChannelCredentials for use with an ALTS-enabled Channel. This is an EXPERIMENTAL API. + ALTS credentials API can only be used in GCP environment as it relies on + handshaker service being available. For more info about ALTS see + https://cloud.google.com/security/encryption-in-transit/application-layer-transport-security Args: - service_accounts: list of strings, target service accounts + service_accounts: A list of server identities accepted by the client. + If target service accounts are provided and none of them matches the + peer identity of the server, handshake will fail. The arg can be empty + if the client does not have any information about trusted server + identity. Returns: A ChannelCredentials for use with an ALTS-enabled Channel """ @@ -1847,9 +1854,12 @@ def alts_channel_credentials(service_accounts=[]): def alts_server_credentials(): - """Creates a ServerCredentials for use with an ALTS-enabled connections. + """Creates a ServerCredentials for use with an ALTS-enabled connection. This is an EXPERIMENTAL API. + ALTS credentials API can only be used in GCP environment as it relies on + handshaker service being available. For more info about ALTS see + https://cloud.google.com/security/encryption-in-transit/application-layer-transport-security Returns: A ServerCredentials for use with an ALTS-enabled Server From edbae5d8e66fd72c8ac743dd54fa41319a5e251b Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Wed, 22 Apr 2020 23:01:56 -0700 Subject: [PATCH 11/47] Fix interceptor batch method FailHijackedRecvMessage for async APIs --- include/grpcpp/impl/codegen/call_op_set.h | 61 ++++-- .../grpcpp/impl/codegen/interceptor_common.h | 8 +- .../client_interceptors_end2end_test.cc | 181 ++++++++++++++++-- test/cpp/end2end/interceptors_util.cc | 57 +++++- test/cpp/end2end/interceptors_util.h | 18 ++ 5 files changed, 286 insertions(+), 39 deletions(-) diff --git a/include/grpcpp/impl/codegen/call_op_set.h b/include/grpcpp/impl/codegen/call_op_set.h index e6d7b900ca1..5bfd541c8f4 100644 --- a/include/grpcpp/impl/codegen/call_op_set.h +++ b/include/grpcpp/impl/codegen/call_op_set.h @@ -421,17 +421,14 @@ Status CallOpSendMessage::SendMessagePtr(const M* message) { template class CallOpRecvMessage { public: - CallOpRecvMessage() - : got_message(false), - message_(nullptr), - allow_not_getting_message_(false) {} + CallOpRecvMessage() {} void RecvMessage(R* message) { message_ = message; } // Do not change status if no message is received. void AllowNoMessage() { allow_not_getting_message_ = true; } - bool got_message; + bool got_message = false; protected: void AddOp(grpc_op* ops, size_t* nops) { @@ -444,7 +441,7 @@ class CallOpRecvMessage { } void FinishOp(bool* status) { - if (message_ == nullptr || hijacked_) return; + if (message_ == nullptr) return; if (recv_buf_.Valid()) { if (*status) { got_message = *status = @@ -455,18 +452,20 @@ class CallOpRecvMessage { got_message = false; recv_buf_.Clear(); } - } else { - got_message = false; - if (!allow_not_getting_message_) { - *status = false; + } else if (hijacked_) { + if (!hijacked_recv_message_status_) { + FinishOpRecvMessageFailureHandler(status); } + } else { + FinishOpRecvMessageFailureHandler(status); } } void SetInterceptionHookPoint( InterceptorBatchMethodsImpl* interceptor_methods) { if (message_ == nullptr) return; - interceptor_methods->SetRecvMessage(message_, &got_message); + interceptor_methods->SetRecvMessage(message_, + &hijacked_recv_message_status_); } void SetFinishInterceptionHookPoint( @@ -485,10 +484,19 @@ class CallOpRecvMessage { } private: - R* message_; + // Sets got_message and \a status for a failed recv message op + void FinishOpRecvMessageFailureHandler(bool* status) { + got_message = false; + if (!allow_not_getting_message_) { + *status = false; + } + } + + R* message_ = nullptr; ByteBuffer recv_buf_; - bool allow_not_getting_message_; + bool allow_not_getting_message_ = false; bool hijacked_ = false; + bool hijacked_recv_message_status_ = true; }; class DeserializeFunc { @@ -513,8 +521,7 @@ class DeserializeFuncType final : public DeserializeFunc { class CallOpGenericRecvMessage { public: - CallOpGenericRecvMessage() - : got_message(false), allow_not_getting_message_(false) {} + CallOpGenericRecvMessage() {} template void RecvMessage(R* message) { @@ -528,7 +535,7 @@ class CallOpGenericRecvMessage { // Do not change status if no message is received. void AllowNoMessage() { allow_not_getting_message_ = true; } - bool got_message; + bool got_message = false; protected: void AddOp(grpc_op* ops, size_t* nops) { @@ -551,6 +558,10 @@ class CallOpGenericRecvMessage { got_message = false; recv_buf_.Clear(); } + } else if (hijacked_) { + if (!hijacked_recv_message_status_) { + FinishOpRecvMessageFailureHandler(status); + } } else { got_message = false; if (!allow_not_getting_message_) { @@ -562,7 +573,8 @@ class CallOpGenericRecvMessage { void SetInterceptionHookPoint( InterceptorBatchMethodsImpl* interceptor_methods) { if (!deserialize_) return; - interceptor_methods->SetRecvMessage(message_, &got_message); + interceptor_methods->SetRecvMessage(message_, + &hijacked_recv_message_status_); } void SetFinishInterceptionHookPoint( @@ -582,11 +594,20 @@ class CallOpGenericRecvMessage { } private: - void* message_; - bool hijacked_ = false; + // Sets got_message and \a status for a failed recv message op + void FinishOpRecvMessageFailureHandler(bool* status) { + got_message = false; + if (!allow_not_getting_message_) { + *status = false; + } + } + + void* message_ = nullptr; std::unique_ptr deserialize_; ByteBuffer recv_buf_; - bool allow_not_getting_message_; + bool allow_not_getting_message_ = false; + bool hijacked_ = false; + bool hijacked_recv_message_status_ = true; }; class CallOpClientSendClose { diff --git a/include/grpcpp/impl/codegen/interceptor_common.h b/include/grpcpp/impl/codegen/interceptor_common.h index 01ffe19bc4a..9b124197ff0 100644 --- a/include/grpcpp/impl/codegen/interceptor_common.h +++ b/include/grpcpp/impl/codegen/interceptor_common.h @@ -166,9 +166,9 @@ class InterceptorBatchMethodsImpl send_trailing_metadata_ = metadata; } - void SetRecvMessage(void* message, bool* got_message) { + void SetRecvMessage(void* message, bool* hijacked_recv_message_status) { recv_message_ = message; - got_message_ = got_message; + hijacked_recv_message_status_ = hijacked_recv_message_status; } void SetRecvInitialMetadata(MetadataMap* map) { @@ -195,7 +195,7 @@ class InterceptorBatchMethodsImpl void FailHijackedRecvMessage() override { GPR_CODEGEN_ASSERT(hooks_[static_cast( experimental::InterceptionHookPoints::PRE_RECV_MESSAGE)]); - *got_message_ = false; + *hijacked_recv_message_status_ = false; } // Clears all state @@ -407,7 +407,7 @@ class InterceptorBatchMethodsImpl std::multimap* send_trailing_metadata_ = nullptr; void* recv_message_ = nullptr; - bool* got_message_ = nullptr; + bool* hijacked_recv_message_status_ = nullptr; MetadataMap* recv_initial_metadata_ = nullptr; diff --git a/test/cpp/end2end/client_interceptors_end2end_test.cc b/test/cpp/end2end/client_interceptors_end2end_test.cc index 7ee95a60c1f..86012a34586 100644 --- a/test/cpp/end2end/client_interceptors_end2end_test.cc +++ b/test/cpp/end2end/client_interceptors_end2end_test.cc @@ -43,6 +43,17 @@ namespace grpc { namespace testing { namespace { +enum class RPCType { + kSyncUnary, + kSyncClientStreaming, + kSyncServerStreaming, + kSyncBidiStreaming, + kAsyncCQUnary, + kAsyncCQClientStreaming, + kAsyncCQServerStreaming, + kAsyncCQBidiStreaming, +}; + /* Hijacks Echo RPC and fills in the expected values */ class HijackingInterceptor : public experimental::Interceptor { public: @@ -400,6 +411,7 @@ class ServerStreamingRpcHijackingInterceptor public: ServerStreamingRpcHijackingInterceptor(experimental::ClientRpcInfo* info) { info_ = info; + got_failed_message_ = false; } virtual void Intercept(experimental::InterceptorBatchMethods* methods) { @@ -531,10 +543,22 @@ class LoggingInterceptor : public experimental::Interceptor { if (methods->QueryInterceptionHookPoint( experimental::InterceptionHookPoints::PRE_SEND_MESSAGE)) { EchoRequest req; - EXPECT_EQ(static_cast(methods->GetSendMessage()) - ->message() - .find("Hello"), - 0u); + auto* send_msg = methods->GetSendMessage(); + if (send_msg == nullptr) { + // We did not get the non-serialized form of the message. Get the + // serialized form. + auto* buffer = methods->GetSerializedSendMessage(); + auto copied_buffer = *buffer; + EchoRequest req; + EXPECT_TRUE( + SerializationTraits::Deserialize(&copied_buffer, &req) + .ok()); + EXPECT_EQ(req.message(), "Hello"); + } else { + EXPECT_EQ( + static_cast(send_msg)->message().find("Hello"), + 0u); + } auto* buffer = methods->GetSerializedSendMessage(); auto copied_buffer = *buffer; EXPECT_TRUE( @@ -582,6 +606,27 @@ class LoggingInterceptor : public experimental::Interceptor { methods->Proceed(); } + static void VerifyCall(RPCType type) { + switch (type) { + case RPCType::kSyncUnary: + case RPCType::kAsyncCQUnary: + VerifyUnaryCall(); + break; + case RPCType::kSyncClientStreaming: + case RPCType::kAsyncCQClientStreaming: + VerifyClientStreamingCall(); + break; + case RPCType::kSyncServerStreaming: + case RPCType::kAsyncCQServerStreaming: + VerifyServerStreamingCall(); + break; + case RPCType::kSyncBidiStreaming: + case RPCType::kAsyncCQBidiStreaming: + VerifyBidiStreamingCall(); + break; + } + } + static void VerifyCallCommon() { EXPECT_TRUE(pre_send_initial_metadata_); EXPECT_TRUE(pre_send_close_); @@ -638,9 +683,31 @@ class LoggingInterceptorFactory } }; -class ClientInterceptorsEnd2endTest : public ::testing::Test { +class TestScenario { + public: + explicit TestScenario(const RPCType& type) : type_(type) {} + + RPCType type() const { return type_; } + + private: + RPCType type_; +}; + +std::vector CreateTestScenarios() { + std::vector scenarios; + scenarios.emplace_back(RPCType::kSyncUnary); + scenarios.emplace_back(RPCType::kSyncClientStreaming); + scenarios.emplace_back(RPCType::kSyncServerStreaming); + scenarios.emplace_back(RPCType::kSyncBidiStreaming); + scenarios.emplace_back(RPCType::kAsyncCQUnary); + scenarios.emplace_back(RPCType::kAsyncCQServerStreaming); + return scenarios; +} + +class ParameterizedClientInterceptorsEnd2endTest + : public ::testing::TestWithParam { protected: - ClientInterceptorsEnd2endTest() { + ParameterizedClientInterceptorsEnd2endTest() { int port = grpc_pick_unused_port_or_die(); ServerBuilder builder; @@ -650,14 +717,44 @@ class ClientInterceptorsEnd2endTest : public ::testing::Test { server_ = builder.BuildAndStart(); } - ~ClientInterceptorsEnd2endTest() { server_->Shutdown(); } + ~ParameterizedClientInterceptorsEnd2endTest() { server_->Shutdown(); } + + void SendRPC(const std::shared_ptr& channel) { + switch (GetParam().type()) { + case RPCType::kSyncUnary: + MakeCall(channel); + break; + case RPCType::kSyncClientStreaming: + MakeClientStreamingCall(channel); + break; + case RPCType::kSyncServerStreaming: + MakeServerStreamingCall(channel); + break; + case RPCType::kSyncBidiStreaming: + MakeBidiStreamingCall(channel); + break; + case RPCType::kAsyncCQUnary: + MakeAsyncCQCall(channel); + break; + case RPCType::kAsyncCQClientStreaming: + // TODO(yashykt) : Fill this out + break; + case RPCType::kAsyncCQServerStreaming: + MakeAsyncCQServerStreamingCall(channel); + break; + case RPCType::kAsyncCQBidiStreaming: + // TODO(yashykt) : Fill this out + break; + } + } std::string server_address_; - TestServiceImpl service_; + EchoTestServiceStreamingImpl service_; std::unique_ptr server_; }; -TEST_F(ClientInterceptorsEnd2endTest, ClientInterceptorLoggingTest) { +TEST_P(ParameterizedClientInterceptorsEnd2endTest, + ClientInterceptorLoggingTest) { ChannelArguments args; DummyInterceptor::Reset(); std::vector> @@ -671,12 +768,36 @@ TEST_F(ClientInterceptorsEnd2endTest, ClientInterceptorLoggingTest) { } auto channel = experimental::CreateCustomChannelWithInterceptors( server_address_, InsecureChannelCredentials(), args, std::move(creators)); - MakeCall(channel); - LoggingInterceptor::VerifyUnaryCall(); + SendRPC(channel); + LoggingInterceptor::VerifyCall(GetParam().type()); // Make sure all 20 dummy interceptors were run EXPECT_EQ(DummyInterceptor::GetNumTimesRun(), 20); } +INSTANTIATE_TEST_SUITE_P(ParameterizedClientInterceptorsEnd2end, + ParameterizedClientInterceptorsEnd2endTest, + ::testing::ValuesIn(CreateTestScenarios())); + +class ClientInterceptorsEnd2endTest + : public ::testing::TestWithParam { + protected: + ClientInterceptorsEnd2endTest() { + int port = grpc_pick_unused_port_or_die(); + + ServerBuilder builder; + server_address_ = "localhost:" + std::to_string(port); + builder.AddListeningPort(server_address_, InsecureServerCredentials()); + builder.RegisterService(&service_); + server_ = builder.BuildAndStart(); + } + + ~ClientInterceptorsEnd2endTest() { server_->Shutdown(); } + + std::string server_address_; + TestServiceImpl service_; + std::unique_ptr server_; +}; + TEST_F(ClientInterceptorsEnd2endTest, LameChannelClientInterceptorHijackingTest) { ChannelArguments args; @@ -757,7 +878,26 @@ TEST_F(ClientInterceptorsEnd2endTest, EXPECT_EQ(DummyInterceptor::GetNumTimesRun(), 12); } -TEST_F(ClientInterceptorsEnd2endTest, +class ClientInterceptorsCallbackEnd2endTest : public ::testing::Test { + protected: + ClientInterceptorsCallbackEnd2endTest() { + int port = grpc_pick_unused_port_or_die(); + + ServerBuilder builder; + server_address_ = "localhost:" + std::to_string(port); + builder.AddListeningPort(server_address_, InsecureServerCredentials()); + builder.RegisterService(&service_); + server_ = builder.BuildAndStart(); + } + + ~ClientInterceptorsCallbackEnd2endTest() { server_->Shutdown(); } + + std::string server_address_; + TestServiceImpl service_; + std::unique_ptr server_; +}; + +TEST_F(ClientInterceptorsCallbackEnd2endTest, ClientInterceptorLoggingTestWithCallback) { ChannelArguments args; DummyInterceptor::Reset(); @@ -778,7 +918,7 @@ TEST_F(ClientInterceptorsEnd2endTest, EXPECT_EQ(DummyInterceptor::GetNumTimesRun(), 20); } -TEST_F(ClientInterceptorsEnd2endTest, +TEST_F(ClientInterceptorsCallbackEnd2endTest, ClientInterceptorFactoryAllowsNullptrReturn) { ChannelArguments args; DummyInterceptor::Reset(); @@ -903,6 +1043,21 @@ TEST_F(ClientInterceptorsStreamingEnd2endTest, ServerStreamingHijackingTest) { EXPECT_TRUE(ServerStreamingRpcHijackingInterceptor::GotFailedMessage()); } +TEST_F(ClientInterceptorsStreamingEnd2endTest, + AsyncCQServerStreamingHijackingTest) { + ChannelArguments args; + DummyInterceptor::Reset(); + std::vector> + creators; + creators.push_back( + std::unique_ptr( + new ServerStreamingRpcHijackingInterceptorFactory())); + auto channel = experimental::CreateCustomChannelWithInterceptors( + server_address_, InsecureChannelCredentials(), args, std::move(creators)); + MakeAsyncCQServerStreamingCall(channel); + EXPECT_TRUE(ServerStreamingRpcHijackingInterceptor::GotFailedMessage()); +} + TEST_F(ClientInterceptorsStreamingEnd2endTest, BidiStreamingHijackingTest) { ChannelArguments args; DummyInterceptor::Reset(); diff --git a/test/cpp/end2end/interceptors_util.cc b/test/cpp/end2end/interceptors_util.cc index 6321c35ba4a..7d74a48db62 100644 --- a/test/cpp/end2end/interceptors_util.cc +++ b/test/cpp/end2end/interceptors_util.cc @@ -66,7 +66,6 @@ void MakeServerStreamingCall(const std::shared_ptr& channel) { ctx.AddMetadata("testkey", "testvalue"); req.set_message("Hello"); EchoResponse resp; - string expected_resp = ""; auto reader = stub->ResponseStream(&ctx, req); int count = 0; while (reader->Read(&resp)) { @@ -84,6 +83,7 @@ void MakeBidiStreamingCall(const std::shared_ptr& channel) { EchoRequest req; EchoResponse resp; ctx.AddMetadata("testkey", "testvalue"); + req.mutable_param()->set_echo_metadata(true); auto stream = stub->BidiStream(&ctx); for (auto i = 0; i < kNumStreamingMessages; i++) { req.set_message("Hello" + std::to_string(i)); @@ -96,6 +96,60 @@ void MakeBidiStreamingCall(const std::shared_ptr& channel) { EXPECT_EQ(s.ok(), true); } +void MakeAsyncCQCall(const std::shared_ptr& channel) { + auto stub = grpc::testing::EchoTestService::NewStub(channel); + CompletionQueue cq; + EchoRequest send_request; + EchoResponse recv_response; + Status recv_status; + ClientContext cli_ctx; + + send_request.set_message("Hello"); + cli_ctx.AddMetadata("testkey", "testvalue"); + std::unique_ptr> response_reader( + stub->AsyncEcho(&cli_ctx, send_request, &cq)); + response_reader->Finish(&recv_response, &recv_status, tag(1)); + Verifier().Expect(1, true).Verify(&cq); + EXPECT_EQ(send_request.message(), recv_response.message()); + EXPECT_TRUE(recv_status.ok()); +} + +void MakeAsyncCQClientStreamingCall(const std::shared_ptr& channel) { + // TODO(yashykt) : Fill this out +} + +void MakeAsyncCQServerStreamingCall(const std::shared_ptr& channel) { + auto stub = grpc::testing::EchoTestService::NewStub(channel); + CompletionQueue cq; + EchoRequest send_request; + EchoResponse recv_response; + Status recv_status; + ClientContext cli_ctx; + + cli_ctx.AddMetadata("testkey", "testvalue"); + send_request.set_message("Hello"); + std::unique_ptr> cli_stream( + stub->AsyncResponseStream(&cli_ctx, send_request, &cq, tag(1))); + Verifier().Expect(1, true).Verify(&cq); + // Read the expected number of messages + for (int i = 0; i < kNumStreamingMessages; i++) { + cli_stream->Read(&recv_response, tag(2)); + Verifier().Expect(2, true).Verify(&cq); + ASSERT_EQ(recv_response.message(), send_request.message()); + } + // The next read should fail + cli_stream->Read(&recv_response, tag(3)); + Verifier().Expect(3, false).Verify(&cq); + // Get the status + cli_stream->Finish(&recv_status, tag(4)); + Verifier().Expect(4, true).Verify(&cq); + EXPECT_TRUE(recv_status.ok()); +} + +void MakeAsyncCQBidiStreamingCall(const std::shared_ptr& channel) { + // TODO(yashykt) : Fill this out +} + void MakeCallbackCall(const std::shared_ptr& channel) { auto stub = grpc::testing::EchoTestService::NewStub(channel); ClientContext ctx; @@ -109,7 +163,6 @@ void MakeCallbackCall(const std::shared_ptr& channel) { EchoResponse resp; stub->experimental_async()->Echo(&ctx, &req, &resp, [&resp, &mu, &done, &cv](Status s) { - // gpr_log(GPR_ERROR, "got the callback"); EXPECT_EQ(s.ok(), true); EXPECT_EQ(resp.message(), "Hello"); std::lock_guard l(mu); diff --git a/test/cpp/end2end/interceptors_util.h b/test/cpp/end2end/interceptors_util.h index 6027c9b3dcf..cac9f3a6bb8 100644 --- a/test/cpp/end2end/interceptors_util.h +++ b/test/cpp/end2end/interceptors_util.h @@ -102,6 +102,16 @@ class EchoTestServiceStreamingImpl : public EchoTestService::Service { public: ~EchoTestServiceStreamingImpl() override {} + Status Echo(ServerContext* context, const EchoRequest* request, + EchoResponse* response) { + auto client_metadata = context->client_metadata(); + for (const auto& pair : client_metadata) { + context->AddTrailingMetadata(ToString(pair.first), ToString(pair.second)); + } + response->set_message(request->message()); + return Status::OK; + } + Status BidiStream( ServerContext* context, grpc::ServerReaderWriter* stream) override { @@ -162,6 +172,14 @@ void MakeServerStreamingCall(const std::shared_ptr& channel); void MakeBidiStreamingCall(const std::shared_ptr& channel); +void MakeAsyncCQCall(const std::shared_ptr& channel); + +void MakeAsyncCQClientStreamingCall(const std::shared_ptr& channel); + +void MakeAsyncCQServerStreamingCall(const std::shared_ptr& channel); + +void MakeAsyncCQBidiStreamingCall(const std::shared_ptr& channel); + void MakeCallbackCall(const std::shared_ptr& channel); bool CheckMetadata(const std::multimap& map, From e714d0fcb876853da7edae71babcae2b0be9659a Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Tue, 28 Apr 2020 13:24:23 -0700 Subject: [PATCH 12/47] Reviewer comments --- include/grpcpp/impl/codegen/call_op_set.h | 20 +++++++++++++------ .../grpcpp/impl/codegen/interceptor_common.h | 8 ++++---- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/include/grpcpp/impl/codegen/call_op_set.h b/include/grpcpp/impl/codegen/call_op_set.h index 5bfd541c8f4..79fa08ad6dc 100644 --- a/include/grpcpp/impl/codegen/call_op_set.h +++ b/include/grpcpp/impl/codegen/call_op_set.h @@ -453,8 +453,12 @@ class CallOpRecvMessage { recv_buf_.Clear(); } } else if (hijacked_) { - if (!hijacked_recv_message_status_) { + if (hijacked_recv_message_failed_) { FinishOpRecvMessageFailureHandler(status); + } else { + // The op was hijacked and it was successful. There is no further action + // to be performed since the message is already in its non-serialized + // form. } } else { FinishOpRecvMessageFailureHandler(status); @@ -465,7 +469,7 @@ class CallOpRecvMessage { InterceptorBatchMethodsImpl* interceptor_methods) { if (message_ == nullptr) return; interceptor_methods->SetRecvMessage(message_, - &hijacked_recv_message_status_); + &hijacked_recv_message_failed_); } void SetFinishInterceptionHookPoint( @@ -496,7 +500,7 @@ class CallOpRecvMessage { ByteBuffer recv_buf_; bool allow_not_getting_message_ = false; bool hijacked_ = false; - bool hijacked_recv_message_status_ = true; + bool hijacked_recv_message_failed_ = false; }; class DeserializeFunc { @@ -559,8 +563,12 @@ class CallOpGenericRecvMessage { recv_buf_.Clear(); } } else if (hijacked_) { - if (!hijacked_recv_message_status_) { + if (hijacked_recv_message_failed_) { FinishOpRecvMessageFailureHandler(status); + } else { + // The op was hijacked and it was successful. There is no further action + // to be performed since the message is already in its non-serialized + // form. } } else { got_message = false; @@ -574,7 +582,7 @@ class CallOpGenericRecvMessage { InterceptorBatchMethodsImpl* interceptor_methods) { if (!deserialize_) return; interceptor_methods->SetRecvMessage(message_, - &hijacked_recv_message_status_); + &hijacked_recv_message_failed_); } void SetFinishInterceptionHookPoint( @@ -607,7 +615,7 @@ class CallOpGenericRecvMessage { ByteBuffer recv_buf_; bool allow_not_getting_message_ = false; bool hijacked_ = false; - bool hijacked_recv_message_status_ = true; + bool hijacked_recv_message_failed_ = false; }; class CallOpClientSendClose { diff --git a/include/grpcpp/impl/codegen/interceptor_common.h b/include/grpcpp/impl/codegen/interceptor_common.h index 9b124197ff0..c87c8034c77 100644 --- a/include/grpcpp/impl/codegen/interceptor_common.h +++ b/include/grpcpp/impl/codegen/interceptor_common.h @@ -166,9 +166,9 @@ class InterceptorBatchMethodsImpl send_trailing_metadata_ = metadata; } - void SetRecvMessage(void* message, bool* hijacked_recv_message_status) { + void SetRecvMessage(void* message, bool* hijacked_recv_message_failed) { recv_message_ = message; - hijacked_recv_message_status_ = hijacked_recv_message_status; + hijacked_recv_message_failed_ = hijacked_recv_message_failed; } void SetRecvInitialMetadata(MetadataMap* map) { @@ -195,7 +195,7 @@ class InterceptorBatchMethodsImpl void FailHijackedRecvMessage() override { GPR_CODEGEN_ASSERT(hooks_[static_cast( experimental::InterceptionHookPoints::PRE_RECV_MESSAGE)]); - *hijacked_recv_message_status_ = false; + *hijacked_recv_message_failed_ = true; } // Clears all state @@ -407,7 +407,7 @@ class InterceptorBatchMethodsImpl std::multimap* send_trailing_metadata_ = nullptr; void* recv_message_ = nullptr; - bool* hijacked_recv_message_status_ = nullptr; + bool* hijacked_recv_message_failed_ = nullptr; MetadataMap* recv_initial_metadata_ = nullptr; From 8c64e96458188000a7c883db1a6014e97888b7c6 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Sat, 25 Apr 2020 17:30:17 -0700 Subject: [PATCH 13/47] Fix flaky LastCallStartedTime --- test/core/channel/channelz_test.cc | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/test/core/channel/channelz_test.cc b/test/core/channel/channelz_test.cc index b16d2d53e0b..338cd0f7ee6 100644 --- a/test/core/channel/channelz_test.cc +++ b/test/core/channel/channelz_test.cc @@ -48,11 +48,11 @@ namespace testing { class CallCountingHelperPeer { public: explicit CallCountingHelperPeer(CallCountingHelper* node) : node_(node) {} - grpc_millis last_call_started_millis() const { + + gpr_timespec last_call_started_time() const { CallCountingHelper::CounterData data; node_->CollectData(&data); - gpr_timespec ts = gpr_cycle_counter_to_time(data.last_call_started_cycle); - return grpc_timespec_to_millis_round_up(ts); + return gpr_cycle_counter_to_time(data.last_call_started_cycle); } private: @@ -243,9 +243,9 @@ void ValidateServer(ServerNode* server, const ValidateChannelDataArgs& args) { gpr_free(core_api_json_str); } -grpc_millis GetLastCallStartedMillis(CallCountingHelper* channel) { +gpr_timespec GetLastCallStartedTime(CallCountingHelper* channel) { CallCountingHelperPeer peer(channel); - return peer.last_call_started_millis(); + return peer.last_call_started_time(); } void ChannelzSleep(int64_t sleep_us) { @@ -301,28 +301,28 @@ TEST_P(ChannelzChannelTest, BasicChannelAPIFunctionality) { ValidateChannel(channelz_channel, {3, 3, 3}); } -TEST_P(ChannelzChannelTest, LastCallStartedMillis) { +TEST_P(ChannelzChannelTest, LastCallStartedTime) { grpc_core::ExecCtx exec_ctx; CallCountingHelper counter; // start a call to set the last call started timestamp counter.RecordCallStarted(); - grpc_millis millis1 = GetLastCallStartedMillis(&counter); + gpr_timespec time1 = GetLastCallStartedTime(&counter); // time gone by should not affect the timestamp ChannelzSleep(100); - grpc_millis millis2 = GetLastCallStartedMillis(&counter); - EXPECT_EQ(millis1, millis2); + gpr_timespec time2 = GetLastCallStartedTime(&counter); + EXPECT_EQ(gpr_time_cmp(time1, time2), 0); // calls succeeded or failed should not affect the timestamp ChannelzSleep(100); counter.RecordCallFailed(); counter.RecordCallSucceeded(); - grpc_millis millis3 = GetLastCallStartedMillis(&counter); - EXPECT_EQ(millis1, millis3); + gpr_timespec time3 = GetLastCallStartedTime(&counter); + EXPECT_EQ(gpr_time_cmp(time1, time3), 0); // another call started should affect the timestamp // sleep for extra long to avoid flakes (since we cache Now()) ChannelzSleep(5000); counter.RecordCallStarted(); - grpc_millis millis4 = GetLastCallStartedMillis(&counter); - EXPECT_NE(millis1, millis4); + gpr_timespec time4 = GetLastCallStartedTime(&counter); + EXPECT_NE(gpr_time_cmp(time1, time4), 0); } class ChannelzRegistryBasedTest : public ::testing::TestWithParam { From 657275fbc01dc41014dc9dab131f5e21dc4144a2 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Wed, 29 Apr 2020 09:56:22 -0700 Subject: [PATCH 14/47] Add note to gpr_convert_clock_type --- src/core/lib/gpr/time.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/core/lib/gpr/time.cc b/src/core/lib/gpr/time.cc index 8927dab5a30..db5a7f64f70 100644 --- a/src/core/lib/gpr/time.cc +++ b/src/core/lib/gpr/time.cc @@ -254,6 +254,10 @@ gpr_timespec gpr_convert_clock_type(gpr_timespec t, gpr_clock_type clock_type) { return gpr_time_add(gpr_now(clock_type), t); } + // If the given input hits this code, the same result is not guaranteed for + // the same input because it relies on `gpr_now` to calculate the difference + // between two different clocks. Please be careful when you want to use this + // function in unit tests. (e.g. https://github.com/grpc/grpc/pull/22655) return gpr_time_add(gpr_now(clock_type), gpr_time_sub(t, gpr_now(t.clock_type))); } From d75de51e43e6d1844c086f338d8ed5a6348f5b35 Mon Sep 17 00:00:00 2001 From: ZhenLian Date: Tue, 28 Apr 2020 23:23:38 -0700 Subject: [PATCH 15/47] [10/n] remove hard-coded credential paths and rebuild the project --- CMakeLists.txt | 24 -------- Makefile | 72 ++---------------------- build_autogenerated.yaml | 38 ++----------- gRPC-Core.podspec | 5 -- grpc.gyp | 8 --- test/core/bad_ssl/generate_tests.bzl | 1 - test/core/bad_ssl/servers/cert.cc | 1 - test/core/end2end/fixtures/h2_fakesec.cc | 1 - test/core/end2end/generate_tests.bzl | 2 - test/core/security/BUILD | 3 - test/core/surface/BUILD | 1 - test/cpp/util/BUILD | 1 - 12 files changed, 8 insertions(+), 149 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ed5d9579c57..71f87ec3574 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -895,10 +895,6 @@ if(gRPC_BUILD_TESTS) add_library(end2end_nosec_tests test/core/end2end/cq_verifier.cc - test/core/end2end/data/client_certs.cc - test/core/end2end/data/server1_cert.cc - test/core/end2end/data/server1_key.cc - test/core/end2end/data/test_root_cert.cc test/core/end2end/end2end_nosec_tests.cc test/core/end2end/end2end_test_utils.cc test/core/end2end/fixtures/http_proxy_fixture.cc @@ -1026,10 +1022,6 @@ if(gRPC_BUILD_TESTS) add_library(end2end_tests test/core/end2end/cq_verifier.cc - test/core/end2end/data/client_certs.cc - test/core/end2end/data/server1_cert.cc - test/core/end2end/data/server1_key.cc - test/core/end2end/data/test_root_cert.cc test/core/end2end/end2end_test_utils.cc test/core/end2end/end2end_tests.cc test/core/end2end/fixtures/http_proxy_fixture.cc @@ -6767,10 +6759,6 @@ endif() if(gRPC_BUILD_TESTS) add_executable(sequential_connectivity_test - test/core/end2end/data/client_certs.cc - test/core/end2end/data/server1_cert.cc - test/core/end2end/data/server1_key.cc - test/core/end2end/data/test_root_cert.cc test/core/surface/sequential_connectivity_test.cc ) @@ -11073,10 +11061,6 @@ endif() if(gRPC_BUILD_TESTS) add_executable(grpc_tls_credentials_options_test - test/core/end2end/data/client_certs.cc - test/core/end2end/data/server1_cert.cc - test/core/end2end/data/server1_key.cc - test/core/end2end/data/test_root_cert.cc test/core/security/grpc_tls_credentials_options_test.cc third_party/googletest/googletest/src/gtest-all.cc third_party/googletest/googlemock/src/gmock-all.cc @@ -14067,10 +14051,6 @@ endif() if(gRPC_BUILD_TESTS) add_executable(tls_security_connector_test - test/core/end2end/data/client_certs.cc - test/core/end2end/data/server1_cert.cc - test/core/end2end/data/server1_key.cc - test/core/end2end/data/test_root_cert.cc test/core/security/tls_security_connector_test.cc third_party/googletest/googletest/src/gtest-all.cc third_party/googletest/googlemock/src/gmock-all.cc @@ -14962,10 +14942,6 @@ endif() if(gRPC_BUILD_TESTS) add_executable(ssl_server_fuzzer_one_entry - test/core/end2end/data/client_certs.cc - test/core/end2end/data/server1_cert.cc - test/core/end2end/data/server1_key.cc - test/core/end2end/data/test_root_cert.cc test/core/security/ssl_server_fuzzer.cc test/core/util/one_corpus_entry_fuzzer.cc third_party/googletest/googletest/src/gtest-all.cc diff --git a/Makefile b/Makefile index 62d0d31011b..175c22a52bc 100644 --- a/Makefile +++ b/Makefile @@ -3258,10 +3258,6 @@ endif LIBEND2END_NOSEC_TESTS_SRC = \ test/core/end2end/cq_verifier.cc \ - test/core/end2end/data/client_certs.cc \ - test/core/end2end/data/server1_cert.cc \ - test/core/end2end/data/server1_key.cc \ - test/core/end2end/data/test_root_cert.cc \ test/core/end2end/end2end_nosec_tests.cc \ test/core/end2end/end2end_test_utils.cc \ test/core/end2end/fixtures/http_proxy_fixture.cc \ @@ -3370,10 +3366,6 @@ endif LIBEND2END_TESTS_SRC = \ test/core/end2end/cq_verifier.cc \ - test/core/end2end/data/client_certs.cc \ - test/core/end2end/data/server1_cert.cc \ - test/core/end2end/data/server1_key.cc \ - test/core/end2end/data/test_root_cert.cc \ test/core/end2end/end2end_test_utils.cc \ test/core/end2end/end2end_tests.cc \ test/core/end2end/fixtures/http_proxy_fixture.cc \ @@ -9999,10 +9991,6 @@ endif SEQUENTIAL_CONNECTIVITY_TEST_SRC = \ - test/core/end2end/data/client_certs.cc \ - test/core/end2end/data/server1_cert.cc \ - test/core/end2end/data/server1_key.cc \ - test/core/end2end/data/test_root_cert.cc \ test/core/surface/sequential_connectivity_test.cc \ SEQUENTIAL_CONNECTIVITY_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(SEQUENTIAL_CONNECTIVITY_TEST_SRC)))) @@ -10023,14 +10011,6 @@ $(BINDIR)/$(CONFIG)/sequential_connectivity_test: $(SEQUENTIAL_CONNECTIVITY_TEST endif -$(OBJDIR)/$(CONFIG)/test/core/end2end/data/client_certs.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a - -$(OBJDIR)/$(CONFIG)/test/core/end2end/data/server1_cert.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a - -$(OBJDIR)/$(CONFIG)/test/core/end2end/data/server1_key.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a - -$(OBJDIR)/$(CONFIG)/test/core/end2end/data/test_root_cert.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a - $(OBJDIR)/$(CONFIG)/test/core/surface/sequential_connectivity_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a deps_sequential_connectivity_test: $(SEQUENTIAL_CONNECTIVITY_TEST_OBJS:.o=.dep) @@ -14701,10 +14681,6 @@ endif GRPC_TLS_CREDENTIALS_OPTIONS_TEST_SRC = \ - test/core/end2end/data/client_certs.cc \ - test/core/end2end/data/server1_cert.cc \ - test/core/end2end/data/server1_key.cc \ - test/core/end2end/data/test_root_cert.cc \ test/core/security/grpc_tls_credentials_options_test.cc \ GRPC_TLS_CREDENTIALS_OPTIONS_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(GRPC_TLS_CREDENTIALS_OPTIONS_TEST_SRC)))) @@ -14736,14 +14712,6 @@ endif endif -$(OBJDIR)/$(CONFIG)/test/core/end2end/data/client_certs.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a - -$(OBJDIR)/$(CONFIG)/test/core/end2end/data/server1_cert.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a - -$(OBJDIR)/$(CONFIG)/test/core/end2end/data/server1_key.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a - -$(OBJDIR)/$(CONFIG)/test/core/end2end/data/test_root_cert.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a - $(OBJDIR)/$(CONFIG)/test/core/security/grpc_tls_credentials_options_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a deps_grpc_tls_credentials_options_test: $(GRPC_TLS_CREDENTIALS_OPTIONS_TEST_OBJS:.o=.dep) @@ -17864,10 +17832,6 @@ endif SSL_SERVER_FUZZER_SRC = \ - test/core/end2end/data/client_certs.cc \ - test/core/end2end/data/server1_cert.cc \ - test/core/end2end/data/server1_key.cc \ - test/core/end2end/data/test_root_cert.cc \ test/core/security/ssl_server_fuzzer.cc \ test/core/util/fuzzer_corpus_test.cc \ @@ -17900,14 +17864,6 @@ endif endif -$(OBJDIR)/$(CONFIG)/test/core/end2end/data/client_certs.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a - -$(OBJDIR)/$(CONFIG)/test/core/end2end/data/server1_cert.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a - -$(OBJDIR)/$(CONFIG)/test/core/end2end/data/server1_key.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a - -$(OBJDIR)/$(CONFIG)/test/core/end2end/data/test_root_cert.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a - $(OBJDIR)/$(CONFIG)/test/core/security/ssl_server_fuzzer.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a $(OBJDIR)/$(CONFIG)/test/core/util/fuzzer_corpus_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a @@ -18507,10 +18463,6 @@ endif TLS_SECURITY_CONNECTOR_TEST_SRC = \ - test/core/end2end/data/client_certs.cc \ - test/core/end2end/data/server1_cert.cc \ - test/core/end2end/data/server1_key.cc \ - test/core/end2end/data/test_root_cert.cc \ test/core/security/tls_security_connector_test.cc \ TLS_SECURITY_CONNECTOR_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(TLS_SECURITY_CONNECTOR_TEST_SRC)))) @@ -18542,14 +18494,6 @@ endif endif -$(OBJDIR)/$(CONFIG)/test/core/end2end/data/client_certs.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a - -$(OBJDIR)/$(CONFIG)/test/core/end2end/data/server1_cert.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a - -$(OBJDIR)/$(CONFIG)/test/core/end2end/data/server1_key.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a - -$(OBJDIR)/$(CONFIG)/test/core/end2end/data/test_root_cert.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a - $(OBJDIR)/$(CONFIG)/test/core/security/tls_security_connector_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a deps_tls_security_connector_test: $(TLS_SECURITY_CONNECTOR_TEST_OBJS:.o=.dep) @@ -19884,10 +19828,6 @@ endif SSL_SERVER_FUZZER_ONE_ENTRY_SRC = \ - test/core/end2end/data/client_certs.cc \ - test/core/end2end/data/server1_cert.cc \ - test/core/end2end/data/server1_key.cc \ - test/core/end2end/data/test_root_cert.cc \ test/core/security/ssl_server_fuzzer.cc \ test/core/util/one_corpus_entry_fuzzer.cc \ @@ -19920,14 +19860,6 @@ endif endif -$(OBJDIR)/$(CONFIG)/test/core/end2end/data/client_certs.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a - -$(OBJDIR)/$(CONFIG)/test/core/end2end/data/server1_cert.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a - -$(OBJDIR)/$(CONFIG)/test/core/end2end/data/server1_key.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a - -$(OBJDIR)/$(CONFIG)/test/core/end2end/data/test_root_cert.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a - $(OBJDIR)/$(CONFIG)/test/core/security/ssl_server_fuzzer.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a $(OBJDIR)/$(CONFIG)/test/core/util/one_corpus_entry_fuzzer.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a @@ -20091,6 +20023,10 @@ src/cpp/server/channelz/channelz_service_plugin.cc: $(OPENSSL_DEP) src/cpp/server/secure_server_credentials.cc: $(OPENSSL_DEP) src/cpp/util/error_details.cc: $(OPENSSL_DEP) src/csharp/ext/grpc_csharp_ext.c: $(OPENSSL_DEP) +test/core/end2end/data/client_certs.cc: $(OPENSSL_DEP) +test/core/end2end/data/server1_cert.cc: $(OPENSSL_DEP) +test/core/end2end/data/server1_key.cc: $(OPENSSL_DEP) +test/core/end2end/data/test_root_cert.cc: $(OPENSSL_DEP) test/core/end2end/end2end_tests.cc: $(OPENSSL_DEP) test/core/end2end/tests/call_creds.cc: $(OPENSSL_DEP) test/cpp/microbenchmarks/helpers.cc: $(OPENSSL_DEP) diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index eeb81843c74..60d8870c93d 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -19,7 +19,6 @@ libs: public_headers: [] headers: - test/core/end2end/cq_verifier.h - - test/core/end2end/data/ssl_test_data.h - test/core/end2end/end2end_tests.h - test/core/end2end/fixtures/http_proxy_fixture.h - test/core/end2end/fixtures/local_util.h @@ -27,10 +26,6 @@ libs: - test/core/end2end/tests/cancel_test_helpers.h src: - test/core/end2end/cq_verifier.cc - - test/core/end2end/data/client_certs.cc - - test/core/end2end/data/server1_cert.cc - - test/core/end2end/data/server1_key.cc - - test/core/end2end/data/test_root_cert.cc - test/core/end2end/end2end_nosec_tests.cc - test/core/end2end/end2end_test_utils.cc - test/core/end2end/fixtures/http_proxy_fixture.cc @@ -127,7 +122,6 @@ libs: public_headers: [] headers: - test/core/end2end/cq_verifier.h - - test/core/end2end/data/ssl_test_data.h - test/core/end2end/end2end_tests.h - test/core/end2end/fixtures/http_proxy_fixture.h - test/core/end2end/fixtures/local_util.h @@ -135,10 +129,6 @@ libs: - test/core/end2end/tests/cancel_test_helpers.h src: - test/core/end2end/cq_verifier.cc - - test/core/end2end/data/client_certs.cc - - test/core/end2end/data/server1_cert.cc - - test/core/end2end/data/server1_key.cc - - test/core/end2end/data/test_root_cert.cc - test/core/end2end/end2end_test_utils.cc - test/core/end2end/end2end_tests.cc - test/core/end2end/fixtures/http_proxy_fixture.cc @@ -4086,13 +4076,8 @@ targets: build: test run: false language: c - headers: - - test/core/end2end/data/ssl_test_data.h + headers: [] src: - - test/core/end2end/data/client_certs.cc - - test/core/end2end/data/server1_cert.cc - - test/core/end2end/data/server1_key.cc - - test/core/end2end/data/test_root_cert.cc - test/core/surface/sequential_connectivity_test.cc deps: - grpc_test_util @@ -5989,13 +5974,8 @@ targets: gtest: true build: test language: c++ - headers: - - test/core/end2end/data/ssl_test_data.h + headers: [] src: - - test/core/end2end/data/client_certs.cc - - test/core/end2end/data/server1_cert.cc - - test/core/end2end/data/server1_key.cc - - test/core/end2end/data/test_root_cert.cc - test/core/security/grpc_tls_credentials_options_test.cc deps: - grpc_test_util @@ -7128,13 +7108,8 @@ targets: - name: ssl_server_fuzzer build: fuzzer language: c++ - headers: - - test/core/end2end/data/ssl_test_data.h + headers: [] src: - - test/core/end2end/data/client_certs.cc - - test/core/end2end/data/server1_cert.cc - - test/core/end2end/data/server1_key.cc - - test/core/end2end/data/test_root_cert.cc - test/core/security/ssl_server_fuzzer.cc - test/core/util/fuzzer_corpus_test.cc deps: @@ -7362,13 +7337,8 @@ targets: gtest: true build: test language: c++ - headers: - - test/core/end2end/data/ssl_test_data.h + headers: [] src: - - test/core/end2end/data/client_certs.cc - - test/core/end2end/data/server1_cert.cc - - test/core/end2end/data/server1_key.cc - - test/core/end2end/data/test_root_cert.cc - test/core/security/tls_security_connector_test.cc deps: - grpc_test_util diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 8bf16911cc6..fe383edf797 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -1452,11 +1452,6 @@ Pod::Spec.new do |s| ss.source_files = 'test/core/end2end/cq_verifier.cc', 'test/core/end2end/cq_verifier.h', - 'test/core/end2end/data/client_certs.cc', - 'test/core/end2end/data/server1_cert.cc', - 'test/core/end2end/data/server1_key.cc', - 'test/core/end2end/data/ssl_test_data.h', - 'test/core/end2end/data/test_root_cert.cc', 'test/core/end2end/end2end_test_utils.cc', 'test/core/end2end/end2end_tests.cc', 'test/core/end2end/end2end_tests.h', diff --git a/grpc.gyp b/grpc.gyp index 865c403fde7..67165946a75 100644 --- a/grpc.gyp +++ b/grpc.gyp @@ -176,10 +176,6 @@ ], 'sources': [ 'test/core/end2end/cq_verifier.cc', - 'test/core/end2end/data/client_certs.cc', - 'test/core/end2end/data/server1_cert.cc', - 'test/core/end2end/data/server1_key.cc', - 'test/core/end2end/data/test_root_cert.cc', 'test/core/end2end/end2end_nosec_tests.cc', 'test/core/end2end/end2end_test_utils.cc', 'test/core/end2end/fixtures/http_proxy_fixture.cc', @@ -277,10 +273,6 @@ ], 'sources': [ 'test/core/end2end/cq_verifier.cc', - 'test/core/end2end/data/client_certs.cc', - 'test/core/end2end/data/server1_cert.cc', - 'test/core/end2end/data/server1_key.cc', - 'test/core/end2end/data/test_root_cert.cc', 'test/core/end2end/end2end_test_utils.cc', 'test/core/end2end/end2end_tests.cc', 'test/core/end2end/fixtures/http_proxy_fixture.cc', diff --git a/test/core/bad_ssl/generate_tests.bzl b/test/core/bad_ssl/generate_tests.bzl index 1c755912ed8..79e82b3093b 100755 --- a/test/core/bad_ssl/generate_tests.bzl +++ b/test/core/bad_ssl/generate_tests.bzl @@ -29,7 +29,6 @@ def grpc_bad_ssl_tests(): deps = [ "//test/core/util:grpc_test_util", "//:grpc", - "//test/core/end2end:ssl_test_data", ], ) for t in BAD_SSL_TESTS: diff --git a/test/core/bad_ssl/servers/cert.cc b/test/core/bad_ssl/servers/cert.cc index 0722d6b57cd..785e10b99bb 100644 --- a/test/core/bad_ssl/servers/cert.cc +++ b/test/core/bad_ssl/servers/cert.cc @@ -25,7 +25,6 @@ #include "src/core/lib/iomgr/load_file.h" #include "test/core/bad_ssl/server_common.h" -#include "test/core/end2end/data/ssl_test_data.h" /* This server will present an untrusted cert to the connecting client, * causing the SSL handshake to fail */ diff --git a/test/core/end2end/fixtures/h2_fakesec.cc b/test/core/end2end/fixtures/h2_fakesec.cc index 5dacbe43ba4..6a8ba43e3d6 100644 --- a/test/core/end2end/fixtures/h2_fakesec.cc +++ b/test/core/end2end/fixtures/h2_fakesec.cc @@ -27,7 +27,6 @@ #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/gprpp/host_port.h" #include "src/core/lib/security/credentials/fake/fake_credentials.h" -#include "test/core/end2end/data/ssl_test_data.h" #include "test/core/util/port.h" #include "test/core/util/test_config.h" diff --git a/test/core/end2end/generate_tests.bzl b/test/core/end2end/generate_tests.bzl index 7eabb251413..b124947f1ee 100755 --- a/test/core/end2end/generate_tests.bzl +++ b/test/core/end2end/generate_tests.bzl @@ -422,7 +422,6 @@ def grpc_end2end_tests(): language = "C++", deps = [ ":cq_verifier", - ":ssl_test_data", ":http_proxy", ":proxy", ":local_util", @@ -497,7 +496,6 @@ def grpc_end2end_nosec_tests(): language = "C++", deps = [ ":cq_verifier", - ":ssl_test_data", ":http_proxy", ":proxy", ":local_util", diff --git a/test/core/security/BUILD b/test/core/security/BUILD index ad698aa68ad..bd94a80ef75 100644 --- a/test/core/security/BUILD +++ b/test/core/security/BUILD @@ -47,7 +47,6 @@ grpc_fuzzer( deps = [ "//:gpr", "//:grpc", - "//test/core/end2end:ssl_test_data", "//test/core/util:grpc_test_util", ], ) @@ -268,7 +267,6 @@ grpc_cc_test( "//:grpc_secure", "//:tsi", "//:tsi_interface", - "//test/core/end2end:ssl_test_data", "//test/core/util:grpc_test_util", ], ) @@ -287,7 +285,6 @@ grpc_cc_test( "//:gpr", "//:grpc", "//:grpc_secure", - "//test/core/end2end:ssl_test_data", "//test/core/util:grpc_test_util", ], ) diff --git a/test/core/surface/BUILD b/test/core/surface/BUILD index cce1fd70902..37fa57e6927 100644 --- a/test/core/surface/BUILD +++ b/test/core/surface/BUILD @@ -148,7 +148,6 @@ grpc_cc_test( deps = [ "//:gpr", "//:grpc", - "//test/core/end2end:ssl_test_data", "//test/core/util:grpc_test_util", ], ) diff --git a/test/cpp/util/BUILD b/test/cpp/util/BUILD index c3695ba0fa6..2a8ad6d9587 100644 --- a/test/cpp/util/BUILD +++ b/test/cpp/util/BUILD @@ -202,7 +202,6 @@ grpc_cc_test( "//src/proto/grpc/testing:echo_messages_proto", "//src/proto/grpc/testing:echo_proto", "//src/proto/grpc/testing:simple_messages_proto", - "//test/core/end2end:ssl_test_data", "//test/core/util:grpc_test_util", ], ) From 4413d412e2c4c074d6e1a87e04b504e272ede24e Mon Sep 17 00:00:00 2001 From: Taras Galkovskyi Date: Wed, 29 Apr 2020 18:26:20 +0000 Subject: [PATCH 16/47] fix referencing variable out of scope that broke tests --- src/python/grpcio_tests/tests/interop/server.py | 6 +++--- src/python/grpcio_tests/tests_aio/interop/server.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/python/grpcio_tests/tests/interop/server.py b/src/python/grpcio_tests/tests/interop/server.py index e389533aa85..c966271e2a3 100644 --- a/src/python/grpcio_tests/tests/interop/server.py +++ b/src/python/grpcio_tests/tests/interop/server.py @@ -45,8 +45,8 @@ def parse_interop_server_arguments(): return parser.parse_args() -def get_server_credentials(): - if args.use_tls: +def get_server_credentials(use_tls): + if use_tls: private_key = resources.private_key() certificate_chain = resources.certificate_chain() return grpc.ssl_server_credentials(((private_key, certificate_chain),)) @@ -63,7 +63,7 @@ def serve(): test_pb2_grpc.add_TestServiceServicer_to_server(service.TestService(), server) if args.use_tls or args.use_alts: - credentials = get_server_credentials() + credentials = get_server_credentials(use_tls) server.add_secure_port('[::]:{}'.format(args.port), credentials) else: server.add_insecure_port('[::]:{}'.format(args.port)) diff --git a/src/python/grpcio_tests/tests_aio/interop/server.py b/src/python/grpcio_tests/tests_aio/interop/server.py index 82c52cebf0c..509abdf0b2f 100644 --- a/src/python/grpcio_tests/tests_aio/interop/server.py +++ b/src/python/grpcio_tests/tests_aio/interop/server.py @@ -31,7 +31,7 @@ async def serve(): args = interop_server_lib.parse_interop_server_arguments() if args.use_tls or args.use_alts: - credentials = interop_server_lib.get_server_credentials() + credentials = interop_server_lib.get_server_credentials(args.use_tls) address, server = await _test_server.start_test_server( port=args.port, secure=True, server_credentials=credentials) else: From 6a045297c2f52c1c7f283ca3485f72985c0a92a2 Mon Sep 17 00:00:00 2001 From: ZhenLian Date: Wed, 29 Apr 2020 12:22:12 -0700 Subject: [PATCH 17/47] remove dependencies that cause failures --- CMakeLists.txt | 8 ++++++++ Makefile | 12 ++++++++---- build_autogenerated.yaml | 10 ++++++++++ gRPC-Core.podspec | 5 +++++ grpc.gyp | 8 ++++++++ test/core/end2end/generate_tests.bzl | 2 ++ 6 files changed, 41 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 71f87ec3574..32f328341d7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -895,6 +895,10 @@ if(gRPC_BUILD_TESTS) add_library(end2end_nosec_tests test/core/end2end/cq_verifier.cc + test/core/end2end/data/client_certs.cc + test/core/end2end/data/server1_cert.cc + test/core/end2end/data/server1_key.cc + test/core/end2end/data/test_root_cert.cc test/core/end2end/end2end_nosec_tests.cc test/core/end2end/end2end_test_utils.cc test/core/end2end/fixtures/http_proxy_fixture.cc @@ -1022,6 +1026,10 @@ if(gRPC_BUILD_TESTS) add_library(end2end_tests test/core/end2end/cq_verifier.cc + test/core/end2end/data/client_certs.cc + test/core/end2end/data/server1_cert.cc + test/core/end2end/data/server1_key.cc + test/core/end2end/data/test_root_cert.cc test/core/end2end/end2end_test_utils.cc test/core/end2end/end2end_tests.cc test/core/end2end/fixtures/http_proxy_fixture.cc diff --git a/Makefile b/Makefile index 175c22a52bc..bbd8ae81ff6 100644 --- a/Makefile +++ b/Makefile @@ -3258,6 +3258,10 @@ endif LIBEND2END_NOSEC_TESTS_SRC = \ test/core/end2end/cq_verifier.cc \ + test/core/end2end/data/client_certs.cc \ + test/core/end2end/data/server1_cert.cc \ + test/core/end2end/data/server1_key.cc \ + test/core/end2end/data/test_root_cert.cc \ test/core/end2end/end2end_nosec_tests.cc \ test/core/end2end/end2end_test_utils.cc \ test/core/end2end/fixtures/http_proxy_fixture.cc \ @@ -3366,6 +3370,10 @@ endif LIBEND2END_TESTS_SRC = \ test/core/end2end/cq_verifier.cc \ + test/core/end2end/data/client_certs.cc \ + test/core/end2end/data/server1_cert.cc \ + test/core/end2end/data/server1_key.cc \ + test/core/end2end/data/test_root_cert.cc \ test/core/end2end/end2end_test_utils.cc \ test/core/end2end/end2end_tests.cc \ test/core/end2end/fixtures/http_proxy_fixture.cc \ @@ -20023,10 +20031,6 @@ src/cpp/server/channelz/channelz_service_plugin.cc: $(OPENSSL_DEP) src/cpp/server/secure_server_credentials.cc: $(OPENSSL_DEP) src/cpp/util/error_details.cc: $(OPENSSL_DEP) src/csharp/ext/grpc_csharp_ext.c: $(OPENSSL_DEP) -test/core/end2end/data/client_certs.cc: $(OPENSSL_DEP) -test/core/end2end/data/server1_cert.cc: $(OPENSSL_DEP) -test/core/end2end/data/server1_key.cc: $(OPENSSL_DEP) -test/core/end2end/data/test_root_cert.cc: $(OPENSSL_DEP) test/core/end2end/end2end_tests.cc: $(OPENSSL_DEP) test/core/end2end/tests/call_creds.cc: $(OPENSSL_DEP) test/cpp/microbenchmarks/helpers.cc: $(OPENSSL_DEP) diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index 60d8870c93d..361a9955fb8 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -19,6 +19,7 @@ libs: public_headers: [] headers: - test/core/end2end/cq_verifier.h + - test/core/end2end/data/ssl_test_data.h - test/core/end2end/end2end_tests.h - test/core/end2end/fixtures/http_proxy_fixture.h - test/core/end2end/fixtures/local_util.h @@ -26,6 +27,10 @@ libs: - test/core/end2end/tests/cancel_test_helpers.h src: - test/core/end2end/cq_verifier.cc + - test/core/end2end/data/client_certs.cc + - test/core/end2end/data/server1_cert.cc + - test/core/end2end/data/server1_key.cc + - test/core/end2end/data/test_root_cert.cc - test/core/end2end/end2end_nosec_tests.cc - test/core/end2end/end2end_test_utils.cc - test/core/end2end/fixtures/http_proxy_fixture.cc @@ -122,6 +127,7 @@ libs: public_headers: [] headers: - test/core/end2end/cq_verifier.h + - test/core/end2end/data/ssl_test_data.h - test/core/end2end/end2end_tests.h - test/core/end2end/fixtures/http_proxy_fixture.h - test/core/end2end/fixtures/local_util.h @@ -129,6 +135,10 @@ libs: - test/core/end2end/tests/cancel_test_helpers.h src: - test/core/end2end/cq_verifier.cc + - test/core/end2end/data/client_certs.cc + - test/core/end2end/data/server1_cert.cc + - test/core/end2end/data/server1_key.cc + - test/core/end2end/data/test_root_cert.cc - test/core/end2end/end2end_test_utils.cc - test/core/end2end/end2end_tests.cc - test/core/end2end/fixtures/http_proxy_fixture.cc diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index fe383edf797..8bf16911cc6 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -1452,6 +1452,11 @@ Pod::Spec.new do |s| ss.source_files = 'test/core/end2end/cq_verifier.cc', 'test/core/end2end/cq_verifier.h', + 'test/core/end2end/data/client_certs.cc', + 'test/core/end2end/data/server1_cert.cc', + 'test/core/end2end/data/server1_key.cc', + 'test/core/end2end/data/ssl_test_data.h', + 'test/core/end2end/data/test_root_cert.cc', 'test/core/end2end/end2end_test_utils.cc', 'test/core/end2end/end2end_tests.cc', 'test/core/end2end/end2end_tests.h', diff --git a/grpc.gyp b/grpc.gyp index 67165946a75..865c403fde7 100644 --- a/grpc.gyp +++ b/grpc.gyp @@ -176,6 +176,10 @@ ], 'sources': [ 'test/core/end2end/cq_verifier.cc', + 'test/core/end2end/data/client_certs.cc', + 'test/core/end2end/data/server1_cert.cc', + 'test/core/end2end/data/server1_key.cc', + 'test/core/end2end/data/test_root_cert.cc', 'test/core/end2end/end2end_nosec_tests.cc', 'test/core/end2end/end2end_test_utils.cc', 'test/core/end2end/fixtures/http_proxy_fixture.cc', @@ -273,6 +277,10 @@ ], 'sources': [ 'test/core/end2end/cq_verifier.cc', + 'test/core/end2end/data/client_certs.cc', + 'test/core/end2end/data/server1_cert.cc', + 'test/core/end2end/data/server1_key.cc', + 'test/core/end2end/data/test_root_cert.cc', 'test/core/end2end/end2end_test_utils.cc', 'test/core/end2end/end2end_tests.cc', 'test/core/end2end/fixtures/http_proxy_fixture.cc', diff --git a/test/core/end2end/generate_tests.bzl b/test/core/end2end/generate_tests.bzl index b124947f1ee..7eabb251413 100755 --- a/test/core/end2end/generate_tests.bzl +++ b/test/core/end2end/generate_tests.bzl @@ -422,6 +422,7 @@ def grpc_end2end_tests(): language = "C++", deps = [ ":cq_verifier", + ":ssl_test_data", ":http_proxy", ":proxy", ":local_util", @@ -496,6 +497,7 @@ def grpc_end2end_nosec_tests(): language = "C++", deps = [ ":cq_verifier", + ":ssl_test_data", ":http_proxy", ":proxy", ":local_util", From 486e27e2ded4288a17fe6587998248cc96de6ba1 Mon Sep 17 00:00:00 2001 From: Taras Galkovskyi Date: Wed, 29 Apr 2020 19:43:45 +0000 Subject: [PATCH 18/47] fix formatting issues --- examples/python/data_transmission/BUILD | 14 +++++++------- examples/python/data_transmission/alts_client.py | 3 ++- src/python/grpcio_tests/tests/interop/server.py | 2 +- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/examples/python/data_transmission/BUILD b/examples/python/data_transmission/BUILD index 528cd325a1b..7c2980d5a5b 100644 --- a/examples/python/data_transmission/BUILD +++ b/examples/python/data_transmission/BUILD @@ -16,17 +16,17 @@ licenses(["notice"]) # 3-clause BSD load("@grpc_python_dependencies//:requirements.bzl", "requirement") -py_binary( +py_binary( name = "alts_server", - python_version = "PY3", - srcs_version = "PY2AND3", - main = "alts_server.py", srcs = [ "alts_server.py", "demo_pb2.py", "demo_pb2_grpc.py", "server.py", ], + main = "alts_server.py", + python_version = "PY3", + srcs_version = "PY2AND3", deps = [ "//src/python/grpcio/grpc:grpcio", ], @@ -34,15 +34,15 @@ py_binary( py_binary( name = "alts_client", - python_version = "PY3", - srcs_version = "PY2AND3", - main = "alts_client.py", srcs = [ "alts_client.py", "client.py", "demo_pb2.py", "demo_pb2_grpc.py", ], + main = "alts_client.py", + python_version = "PY3", + srcs_version = "PY2AND3", deps = [ "//src/python/grpcio/grpc:grpcio", ], diff --git a/examples/python/data_transmission/alts_client.py b/examples/python/data_transmission/alts_client.py index 0bfcede9465..a2ff05e3646 100644 --- a/examples/python/data_transmission/alts_client.py +++ b/examples/python/data_transmission/alts_client.py @@ -23,7 +23,8 @@ import demo_pb2_grpc SERVER_ADDRESS = "localhost:23333" def main(): - with grpc.secure_channel(SERVER_ADDRESS, credentials=grpc.alts_channel_credentials()) as channel: + with grpc.secure_channel( + SERVER_ADDRESS, credentials=grpc.alts_channel_credentials()) as channel: stub = demo_pb2_grpc.GRPCDemoStub(channel) client.simple_method(stub) client.client_streaming_method(stub) diff --git a/src/python/grpcio_tests/tests/interop/server.py b/src/python/grpcio_tests/tests/interop/server.py index c966271e2a3..a72294aeac8 100644 --- a/src/python/grpcio_tests/tests/interop/server.py +++ b/src/python/grpcio_tests/tests/interop/server.py @@ -63,7 +63,7 @@ def serve(): test_pb2_grpc.add_TestServiceServicer_to_server(service.TestService(), server) if args.use_tls or args.use_alts: - credentials = get_server_credentials(use_tls) + credentials = get_server_credentials(args.use_tls) server.add_secure_port('[::]:{}'.format(args.port), credentials) else: server.add_insecure_port('[::]:{}'.format(args.port)) From 32acd9d5fd2c9a5ce4530f964ca7405a17511757 Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Wed, 29 Apr 2020 22:55:50 +0200 Subject: [PATCH 19/47] [Aio] Stream Unary client interceptor Add support for Stream Unary client interceptor --- .../grpcio/grpc/experimental/aio/__init__.py | 4 +- .../grpcio/grpc/experimental/aio/_call.py | 11 +- .../grpcio/grpc/experimental/aio/_channel.py | 37 +- .../grpc/experimental/aio/_interceptor.py | 211 ++++++- src/python/grpcio_tests/tests_aio/tests.json | 1 + .../client_stream_unary_interceptor_test.py | 531 ++++++++++++++++++ 6 files changed, 769 insertions(+), 26 deletions(-) create mode 100644 src/python/grpcio_tests/tests_aio/unit/client_stream_unary_interceptor_test.py diff --git a/src/python/grpcio/grpc/experimental/aio/__init__.py b/src/python/grpcio/grpc/experimental/aio/__init__.py index d0b6a58a149..c0fc17eeb92 100644 --- a/src/python/grpcio/grpc/experimental/aio/__init__.py +++ b/src/python/grpcio/grpc/experimental/aio/__init__.py @@ -33,7 +33,8 @@ from ._call import AioRpcError from ._interceptor import (ClientCallDetails, ClientInterceptor, InterceptedUnaryUnaryCall, UnaryUnaryClientInterceptor, - UnaryStreamClientInterceptor, ServerInterceptor) + UnaryStreamClientInterceptor, + StreamUnaryClientInterceptor, ServerInterceptor) from ._server import server from ._base_server import Server, ServicerContext from ._typing import ChannelArgumentType @@ -61,6 +62,7 @@ __all__ = ( 'ClientInterceptor', 'UnaryStreamClientInterceptor', 'UnaryUnaryClientInterceptor', + 'StreamUnaryClientInterceptor', 'InterceptedUnaryUnaryCall', 'ServerInterceptor', 'insecure_channel', diff --git a/src/python/grpcio/grpc/experimental/aio/_call.py b/src/python/grpcio/grpc/experimental/aio/_call.py index 00778184658..9cab7052318 100644 --- a/src/python/grpcio/grpc/experimental/aio/_call.py +++ b/src/python/grpcio/grpc/experimental/aio/_call.py @@ -35,6 +35,7 @@ _LOCAL_CANCELLATION_DETAILS = 'Locally cancelled by application!' _GC_CANCELLATION_DETAILS = 'Cancelled upon garbage collection!' _RPC_ALREADY_FINISHED_DETAILS = 'RPC already finished.' _RPC_HALF_CLOSED_DETAILS = 'RPC is half closed after calling "done_writing".' +_API_STYLE_ERROR = 'Please don\'t mix two styles of API for streaming requests' _OK_CALL_REPRESENTATION = ('<{} of RPC that terminated with:\n' '\tstatus = {}\n' @@ -302,8 +303,7 @@ class _StreamResponseMixin(Call): if self._response_style is _APIStyle.UNKNOWN: self._response_style = style elif self._response_style is not style: - raise cygrpc.UsageError( - 'Please don\'t mix two styles of API for streaming responses') + raise cygrpc.UsageError(_API_STYLE_ERROR) def cancel(self) -> bool: if super().cancel(): @@ -381,8 +381,7 @@ class _StreamRequestMixin(Call): def _raise_for_different_style(self, style: _APIStyle): if self._request_style is not style: - raise cygrpc.UsageError( - 'Please don\'t mix two styles of API for streaming requests') + raise cygrpc.UsageError(_API_STYLE_ERROR) def cancel(self) -> bool: if super().cancel(): @@ -399,7 +398,8 @@ class _StreamRequestMixin(Call): request_iterator: RequestIterableType ) -> None: try: - if inspect.isasyncgen(request_iterator): + if inspect.isasyncgen(request_iterator) or hasattr( + request_iterator, '__aiter__'): async for request in request_iterator: await self._write(request) else: @@ -426,7 +426,6 @@ class _StreamRequestMixin(Call): serialized_request = _common.serialize(request, self._request_serializer) - try: await self._cython_call.send_serialized_message(serialized_request) except asyncio.CancelledError: diff --git a/src/python/grpcio/grpc/experimental/aio/_channel.py b/src/python/grpcio/grpc/experimental/aio/_channel.py index afa9cf30630..731a3351b48 100644 --- a/src/python/grpcio/grpc/experimental/aio/_channel.py +++ b/src/python/grpcio/grpc/experimental/aio/_channel.py @@ -25,9 +25,11 @@ from . import _base_call, _base_channel from ._call import (StreamStreamCall, StreamUnaryCall, UnaryStreamCall, UnaryUnaryCall) from ._interceptor import (InterceptedUnaryUnaryCall, - InterceptedUnaryStreamCall, ClientInterceptor, + InterceptedUnaryStreamCall, + InterceptedStreamUnaryCall, ClientInterceptor, UnaryUnaryClientInterceptor, - UnaryStreamClientInterceptor) + UnaryStreamClientInterceptor, + StreamUnaryClientInterceptor) from ._typing import (ChannelArgumentType, DeserializingFunction, MetadataType, SerializingFunction, RequestIterableType) from ._utils import _timeout_to_deadline @@ -167,10 +169,17 @@ class StreamUnaryMultiCallable(_BaseMultiCallable, deadline = _timeout_to_deadline(timeout) - call = StreamUnaryCall(request_iterator, deadline, metadata, - credentials, wait_for_ready, self._channel, - self._method, self._request_serializer, - self._response_deserializer, self._loop) + if not self._interceptors: + call = StreamUnaryCall(request_iterator, deadline, metadata, + credentials, wait_for_ready, self._channel, + self._method, self._request_serializer, + self._response_deserializer, self._loop) + else: + call = InterceptedStreamUnaryCall( + self._interceptors, request_iterator, deadline, metadata, + credentials, wait_for_ready, self._channel, self._method, + self._request_serializer, self._response_deserializer, + self._loop) return call @@ -204,6 +213,7 @@ class Channel(_base_channel.Channel): _channel: cygrpc.AioChannel _unary_unary_interceptors: List[UnaryUnaryClientInterceptor] _unary_stream_interceptors: List[UnaryStreamClientInterceptor] + _stream_unary_interceptors: List[StreamUnaryClientInterceptor] def __init__(self, target: str, options: ChannelArgumentType, credentials: Optional[grpc.ChannelCredentials], @@ -222,12 +232,15 @@ class Channel(_base_channel.Channel): """ self._unary_unary_interceptors = [] self._unary_stream_interceptors = [] + self._stream_unary_interceptors = [] if interceptors: attrs_and_interceptor_classes = ((self._unary_unary_interceptors, UnaryUnaryClientInterceptor), (self._unary_stream_interceptors, - UnaryStreamClientInterceptor)) + UnaryStreamClientInterceptor), + (self._stream_unary_interceptors, + StreamUnaryClientInterceptor)) # pylint: disable=cell-var-from-loop for attr, interceptor_class in attrs_and_interceptor_classes: @@ -238,13 +251,15 @@ class Channel(_base_channel.Channel): invalid_interceptors = set(interceptors) - set( self._unary_unary_interceptors) - set( - self._unary_stream_interceptors) + self._unary_stream_interceptors) - set( + self._stream_unary_interceptors) if invalid_interceptors: raise ValueError( "Interceptor must be "+\ "UnaryUnaryClientInterceptors or "+\ - "UnaryStreamClientInterceptors. The following are invalid: {}"\ + "UnaryUnaryClientInterceptors or "+\ + "StreamUnaryClientInterceptors. The following are invalid: {}"\ .format(invalid_interceptors)) self._loop = asyncio.get_event_loop() @@ -383,7 +398,9 @@ class Channel(_base_channel.Channel): ) -> StreamUnaryMultiCallable: return StreamUnaryMultiCallable(self._channel, _common.encode(method), request_serializer, - response_deserializer, None, self._loop) + response_deserializer, + self._stream_unary_interceptors, + self._loop) def stream_stream( self, diff --git a/src/python/grpcio/grpc/experimental/aio/_interceptor.py b/src/python/grpcio/grpc/experimental/aio/_interceptor.py index b9f786e5522..6a9f8434a0a 100644 --- a/src/python/grpcio/grpc/experimental/aio/_interceptor.py +++ b/src/python/grpcio/grpc/experimental/aio/_interceptor.py @@ -22,10 +22,13 @@ import grpc from grpc._cython import cygrpc from . import _base_call -from ._call import UnaryUnaryCall, UnaryStreamCall, AioRpcError +from ._call import UnaryUnaryCall, UnaryStreamCall, StreamUnaryCall, AioRpcError +from ._call import _RPC_ALREADY_FINISHED_DETAILS, _RPC_HALF_CLOSED_DETAILS +from ._call import _API_STYLE_ERROR from ._utils import _timeout_to_deadline from ._typing import (RequestType, SerializingFunction, DeserializingFunction, - MetadataType, ResponseType, DoneCallbackType) + MetadataType, ResponseType, DoneCallbackType, + RequestIterableType) _LOCAL_CANCELLATION_DETAILS = 'Locally cancelled by application!' @@ -132,13 +135,17 @@ class UnaryStreamClientInterceptor(ClientInterceptor, metaclass=ABCMeta): ) -> Union[AsyncIterable[ResponseType], UnaryStreamCall]: """Intercepts a unary-stream invocation asynchronously. + The function could return the call object or an asynchronous + iterator, in case of being an asyncrhonous iterator this will + become the source of the reads done by the caller. + Args: continuation: A coroutine that proceeds with the invocation by executing the next interceptor in chain or invoking the actual RPC on the underlying Channel. It is the interceptor's responsibility to call it if it decides to move the RPC forward. The interceptor can use - `call = await continuation(client_call_details, request, response_iterator))` + `call = await continuation(client_call_details, request)` to continue with the RPC. `continuation` returns the call to the RPC. client_call_details: A ClientCallDetails object describing the @@ -154,6 +161,42 @@ class UnaryStreamClientInterceptor(ClientInterceptor, metaclass=ABCMeta): """ +class StreamUnaryClientInterceptor(ClientInterceptor, metaclass=ABCMeta): + """Affords intercepting stream-unary invocations.""" + + @abstractmethod + async def intercept_stream_unary( + self, + continuation: Callable[[ClientCallDetails, RequestType], + UnaryStreamCall], + client_call_details: ClientCallDetails, + request_iterator: RequestIterableType, + ) -> StreamUnaryCall: + """Intercepts a stream-unary invocation asynchronously. + + Args: + continuation: A coroutine that proceeds with the invocation by + executing the next interceptor in chain or invoking the + actual RPC on the underlying Channel. It is the interceptor's + responsibility to call it if it decides to move the RPC forward. + The interceptor can use + `call = await continuation(client_call_details, request_iterator)` + to continue with the RPC. `continuation` returns the call to the + RPC. + client_call_details: A ClientCallDetails object describing the + outgoing RPC. + request_iterator: The request iterator that will produce requests + for the RPC. + + Returns: + The RPC Call. + + Raises: + AioRpcError: Indicating that the RPC terminated with non-OK status. + asyncio.CancelledError: Indicating that the RPC was canceled. + """ + + class InterceptedCall: """Base implementation for all intecepted call arities. @@ -332,7 +375,16 @@ class InterceptedCall: return await call.wait_for_connection() -class InterceptedUnaryUnaryCall(InterceptedCall, _base_call.UnaryUnaryCall): +class _InterceptedUnaryResponseMixin: + + def __await__(self): + call = yield from self._interceptors_task.__await__() + response = yield from call.__await__() + return response + + +class InterceptedUnaryUnaryCall(_InterceptedUnaryResponseMixin, InterceptedCall, + _base_call.UnaryUnaryCall): """Used for running a `UnaryUnaryCall` wrapped by interceptors. For the `__await__` method is it is proxied to the intercepted call only when @@ -402,11 +454,6 @@ class InterceptedUnaryUnaryCall(InterceptedCall, _base_call.UnaryUnaryCall): return await _run_interceptor(iter(interceptors), client_call_details, request) - def __await__(self): - call = yield from self._interceptors_task.__await__() - response = yield from call.__await__() - return response - def time_remaining(self) -> Optional[float]: raise NotImplementedError() @@ -504,6 +551,152 @@ class InterceptedUnaryStreamCall(InterceptedCall, _base_call.UnaryStreamCall): raise NotImplementedError() +class InterceptedStreamUnaryCall(_InterceptedUnaryResponseMixin, + InterceptedCall, _base_call.StreamUnaryCall): + """Used for running a `StreamUnaryCall` wrapped by interceptors. + + For the `__await__` method is it is proxied to the intercepted call only when + the interceptor task is finished. + """ + + _loop: asyncio.AbstractEventLoop + _channel: cygrpc.AioChannel + _write_to_iterator_async_gen: Optional[AsyncIterable[RequestType]] + _write_to_iterator_queue: Optional[asyncio.Queue] + + _FINISH_ITERATOR_SENTINEL = tuple() + + # pylint: disable=too-many-arguments + def __init__(self, interceptors: Sequence[StreamUnaryClientInterceptor], + request_iterator: Optional[RequestIterableType], + timeout: Optional[float], metadata: MetadataType, + credentials: Optional[grpc.CallCredentials], + wait_for_ready: Optional[bool], channel: cygrpc.AioChannel, + method: bytes, request_serializer: SerializingFunction, + response_deserializer: DeserializingFunction, + loop: asyncio.AbstractEventLoop) -> None: + self._loop = loop + self._channel = channel + if not request_iterator: + # We provide our own request iterator which is a proxy + # of the future wries done by the caller. This iterator + # will use internally a queue for consuming messages produced + # by the write method. + self._write_to_iterator_queue = asyncio.Queue(maxsize=1) + self._write_to_iterator_async_gen = self._proxies_writes_as_a_request_iteerator( + ) + request_iterator = self._write_to_iterator_async_gen + else: + self._write_to_iterator_queue = None + + interceptors_task = loop.create_task( + self._invoke(interceptors, method, timeout, metadata, credentials, + wait_for_ready, request_iterator, request_serializer, + response_deserializer)) + super().__init__(interceptors_task) + + # pylint: disable=too-many-arguments + async def _invoke( + self, interceptors: Sequence[StreamUnaryClientInterceptor], + method: bytes, timeout: Optional[float], + metadata: Optional[MetadataType], + credentials: Optional[grpc.CallCredentials], + wait_for_ready: Optional[bool], + request_iterator: RequestIterableType, + request_serializer: SerializingFunction, + response_deserializer: DeserializingFunction) -> StreamUnaryCall: + """Run the RPC call wrapped in interceptors""" + + async def _run_interceptor( + interceptors: Iterator[UnaryUnaryClientInterceptor], + client_call_details: ClientCallDetails, + request_iterator: RequestIterableType + ) -> _base_call.StreamUnaryCall: + + interceptor = next(interceptors, None) + + if interceptor: + continuation = functools.partial(_run_interceptor, interceptors) + + return await interceptor.intercept_stream_unary( + continuation, client_call_details, request_iterator) + else: + return StreamUnaryCall( + request_iterator, + _timeout_to_deadline(client_call_details.timeout), + client_call_details.metadata, + client_call_details.credentials, + client_call_details.wait_for_ready, self._channel, + client_call_details.method, request_serializer, + response_deserializer, self._loop) + + client_call_details = ClientCallDetails(method, timeout, metadata, + credentials, wait_for_ready) + return await _run_interceptor(iter(interceptors), client_call_details, + request_iterator) + + def time_remaining(self) -> Optional[float]: + raise NotImplementedError() + + async def _proxies_writes_as_a_request_iteerator(self): + await self._interceptors_task + + while True: + value = await self._write_to_iterator_queue.get() + if value is InterceptedStreamUnaryCall._FINISH_ITERATOR_SENTINEL: + break + yield value + + async def write(self, request: RequestType) -> None: + # If no queue was created it means that requests + # should be expected through an iterators provided + # by the caller. + if self._write_to_iterator_queue is None: + raise cygrpc.UsageError( + 'Please don\'t mix two styles of API for streaming requests') + + try: + call = await self._interceptors_task + except (asyncio.CancelledError, AioRpcError): + raise asyncio.InvalidStateError(_RPC_ALREADY_FINISHED_DETAILS) + + if call.done(): + raise asyncio.InvalidStateError(_RPC_ALREADY_FINISHED_DETAILS) + elif call._done_writing_flag: + raise asyncio.InvalidStateError(_RPC_HALF_CLOSED_DETAILS) + + # Write might never end up since the call could abrubtly finish, + # we give up on the first awaitable object that finishes.. + _, _ = await asyncio.wait( + (self._write_to_iterator_queue.put(request), call), + return_when=asyncio.FIRST_COMPLETED) + + if call.done(): + raise asyncio.InvalidStateError(_RPC_ALREADY_FINISHED_DETAILS) + + async def done_writing(self) -> None: + """Signal peer that client is done writing. + + This method is idempotent. + """ + # If no queue was created it means that requests + # should be expected through an iterators provided + # by the caller. + if self._write_to_iterator_queue is None: + raise cygrpc.UsageError(_API_STYLE_ERROR) + + try: + call = await self._interceptors_task + except asyncio.CancelledError: + raise asyncio.InvalidStateError(_RPC_ALREADY_FINISHED_DETAILS) + + # Write might never end up since the call could abrubtly finish, + # we give up on the first awaitable object that finishes. + _, _ = await asyncio.wait((self._write_to_iterator_queue.put( + InterceptedStreamUnaryCall._FINISH_ITERATOR_SENTINEL), call), + return_when=asyncio.FIRST_COMPLETED) + + class UnaryUnaryCallResponse(_base_call.UnaryUnaryCall): """Final UnaryUnaryCall class finished with a response.""" _response: ResponseType diff --git a/src/python/grpcio_tests/tests_aio/tests.json b/src/python/grpcio_tests/tests_aio/tests.json index 0bdd1f72e50..aeed4363319 100644 --- a/src/python/grpcio_tests/tests_aio/tests.json +++ b/src/python/grpcio_tests/tests_aio/tests.json @@ -13,6 +13,7 @@ "unit.channel_argument_test.TestChannelArgument", "unit.channel_ready_test.TestChannelReady", "unit.channel_test.TestChannel", + "unit.client_stream_unary_interceptor_test.TestStreamUnaryClientInterceptor", "unit.client_unary_stream_interceptor_test.TestUnaryStreamClientInterceptor", "unit.client_unary_unary_interceptor_test.TestInterceptedUnaryUnaryCall", "unit.client_unary_unary_interceptor_test.TestUnaryUnaryClientInterceptor", diff --git a/src/python/grpcio_tests/tests_aio/unit/client_stream_unary_interceptor_test.py b/src/python/grpcio_tests/tests_aio/unit/client_stream_unary_interceptor_test.py new file mode 100644 index 00000000000..318a117ffe4 --- /dev/null +++ b/src/python/grpcio_tests/tests_aio/unit/client_stream_unary_interceptor_test.py @@ -0,0 +1,531 @@ +# Copyright 2020 The gRPC Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import asyncio +import logging +import unittest +import datetime + +import grpc + +from grpc.experimental import aio +from tests_aio.unit._constants import UNREACHABLE_TARGET +from tests_aio.unit._common import inject_callbacks +from tests_aio.unit._test_server import start_test_server +from tests_aio.unit._test_base import AioTestBase +from tests.unit.framework.common import test_constants +from src.proto.grpc.testing import messages_pb2, test_pb2_grpc + +_SHORT_TIMEOUT_S = 1.0 + +_NUM_STREAM_REQUESTS = 5 +_REQUEST_PAYLOAD_SIZE = 7 +_RESPONSE_INTERVAL_US = int(_SHORT_TIMEOUT_S * 1000 * 1000) + + +class _CountingRequestIterator: + + def __init__(self, request_iterator): + self.request_cnt = 0 + self._request_iterator = request_iterator + + async def _forward_requests(self): + async for request in self._request_iterator: + self.request_cnt += 1 + yield request + + def __aiter__(self): + return self._forward_requests() + + +class _StreamUnaryInterceptorEmpty(aio.StreamUnaryClientInterceptor): + + async def intercept_stream_unary(self, continuation, client_call_details, + request_iterator): + return await continuation(client_call_details, request_iterator) + + def assert_in_final_state(self, test: unittest.TestCase): + pass + + +class _StreamUnaryInterceptorWithRequestIterator( + aio.StreamUnaryClientInterceptor): + + async def intercept_stream_unary(self, continuation, client_call_details, + request_iterator): + self.request_iterator = _CountingRequestIterator(request_iterator) + call = await continuation(client_call_details, self.request_iterator) + return call + + def assert_in_final_state(self, test: unittest.TestCase): + test.assertEqual(_NUM_STREAM_REQUESTS, + self.request_iterator.request_cnt) + + +class TestStreamUnaryClientInterceptor(AioTestBase): + + async def setUp(self): + self._server_target, self._server = await start_test_server() + + async def tearDown(self): + await self._server.stop(None) + + async def test_intercepts(self): + for interceptor_class in (_StreamUnaryInterceptorEmpty, + _StreamUnaryInterceptorWithRequestIterator): + + with self.subTest(name=interceptor_class): + interceptor = interceptor_class() + channel = aio.insecure_channel(self._server_target, + interceptors=[interceptor]) + stub = test_pb2_grpc.TestServiceStub(channel) + + payload = messages_pb2.Payload(body=b'\0' * + _REQUEST_PAYLOAD_SIZE) + request = messages_pb2.StreamingInputCallRequest( + payload=payload) + + async def request_iterator(): + for _ in range(_NUM_STREAM_REQUESTS): + yield request + + call = stub.StreamingInputCall(request_iterator()) + + response = await call + + self.assertEqual(_NUM_STREAM_REQUESTS * _REQUEST_PAYLOAD_SIZE, + response.aggregated_payload_size) + self.assertEqual(await call.code(), grpc.StatusCode.OK) + self.assertEqual(await call.initial_metadata(), ()) + self.assertEqual(await call.trailing_metadata(), ()) + self.assertEqual(await call.details(), '') + self.assertEqual(await call.debug_error_string(), '') + self.assertEqual(call.cancel(), False) + self.assertEqual(call.cancelled(), False) + self.assertEqual(call.done(), True) + + interceptor.assert_in_final_state(self) + + await channel.close() + + async def test_intercepts_using_write(self): + for interceptor_class in (_StreamUnaryInterceptorEmpty, + _StreamUnaryInterceptorWithRequestIterator): + + with self.subTest(name=interceptor_class): + interceptor = interceptor_class() + channel = aio.insecure_channel(self._server_target, + interceptors=[interceptor]) + stub = test_pb2_grpc.TestServiceStub(channel) + + payload = messages_pb2.Payload(body=b'\0' * + _REQUEST_PAYLOAD_SIZE) + request = messages_pb2.StreamingInputCallRequest( + payload=payload) + + call = stub.StreamingInputCall() + + for _ in range(_NUM_STREAM_REQUESTS): + await call.write(request) + + await call.done_writing() + + response = await call + + self.assertEqual(_NUM_STREAM_REQUESTS * _REQUEST_PAYLOAD_SIZE, + response.aggregated_payload_size) + self.assertEqual(await call.code(), grpc.StatusCode.OK) + self.assertEqual(await call.initial_metadata(), ()) + self.assertEqual(await call.trailing_metadata(), ()) + self.assertEqual(await call.details(), '') + self.assertEqual(await call.debug_error_string(), '') + self.assertEqual(call.cancel(), False) + self.assertEqual(call.cancelled(), False) + self.assertEqual(call.done(), True) + + interceptor.assert_in_final_state(self) + + await channel.close() + + async def test_add_done_callback_interceptor_task_not_finished(self): + for interceptor_class in (_StreamUnaryInterceptorEmpty, + _StreamUnaryInterceptorWithRequestIterator): + + with self.subTest(name=interceptor_class): + interceptor = interceptor_class() + + channel = aio.insecure_channel(self._server_target, + interceptors=[interceptor]) + stub = test_pb2_grpc.TestServiceStub(channel) + + payload = messages_pb2.Payload(body=b'\0' * + _REQUEST_PAYLOAD_SIZE) + request = messages_pb2.StreamingInputCallRequest( + payload=payload) + + async def request_iterator(): + for _ in range(_NUM_STREAM_REQUESTS): + yield request + + call = stub.StreamingInputCall(request_iterator()) + + validation = inject_callbacks(call) + + response = await call + + await validation + + await channel.close() + + async def test_add_done_callback_interceptor_task_finished(self): + for interceptor_class in (_StreamUnaryInterceptorEmpty, + _StreamUnaryInterceptorWithRequestIterator): + + with self.subTest(name=interceptor_class): + interceptor = interceptor_class() + + channel = aio.insecure_channel(self._server_target, + interceptors=[interceptor]) + stub = test_pb2_grpc.TestServiceStub(channel) + + payload = messages_pb2.Payload(body=b'\0' * + _REQUEST_PAYLOAD_SIZE) + request = messages_pb2.StreamingInputCallRequest( + payload=payload) + + async def request_iterator(): + for _ in range(_NUM_STREAM_REQUESTS): + yield request + + call = stub.StreamingInputCall(request_iterator()) + + response = await call + + validation = inject_callbacks(call) + + await validation + + await channel.close() + + async def test_multiple_interceptors_request_iterator(self): + for interceptor_class in (_StreamUnaryInterceptorEmpty, + _StreamUnaryInterceptorWithRequestIterator): + + with self.subTest(name=interceptor_class): + + interceptors = [interceptor_class(), interceptor_class()] + channel = aio.insecure_channel(self._server_target, + interceptors=interceptors) + stub = test_pb2_grpc.TestServiceStub(channel) + + payload = messages_pb2.Payload(body=b'\0' * + _REQUEST_PAYLOAD_SIZE) + request = messages_pb2.StreamingInputCallRequest( + payload=payload) + + async def request_iterator(): + for _ in range(_NUM_STREAM_REQUESTS): + yield request + + call = stub.StreamingInputCall(request_iterator()) + + response = await call + + self.assertEqual(_NUM_STREAM_REQUESTS * _REQUEST_PAYLOAD_SIZE, + response.aggregated_payload_size) + self.assertEqual(await call.code(), grpc.StatusCode.OK) + self.assertEqual(await call.initial_metadata(), ()) + self.assertEqual(await call.trailing_metadata(), ()) + self.assertEqual(await call.details(), '') + self.assertEqual(await call.debug_error_string(), '') + self.assertEqual(call.cancel(), False) + self.assertEqual(call.cancelled(), False) + self.assertEqual(call.done(), True) + + for interceptor in interceptors: + interceptor.assert_in_final_state(self) + + await channel.close() + + async def test_intercepts_request_iterator_rpc_error(self): + for interceptor_class in (_StreamUnaryInterceptorEmpty, + _StreamUnaryInterceptorWithRequestIterator): + + with self.subTest(name=interceptor_class): + channel = aio.insecure_channel( + UNREACHABLE_TARGET, interceptors=[interceptor_class()]) + stub = test_pb2_grpc.TestServiceStub(channel) + + payload = messages_pb2.Payload(body=b'\0' * + _REQUEST_PAYLOAD_SIZE) + request = messages_pb2.StreamingInputCallRequest( + payload=payload) + + # When there is an error the request iterator is no longer + # consumed. + async def request_iterator(): + for _ in range(_NUM_STREAM_REQUESTS): + yield request + + call = stub.StreamingInputCall(request_iterator()) + + with self.assertRaises(aio.AioRpcError) as exception_context: + await call + + self.assertEqual(grpc.StatusCode.UNAVAILABLE, + exception_context.exception.code()) + self.assertTrue(call.done()) + self.assertEqual(grpc.StatusCode.UNAVAILABLE, await call.code()) + + await channel.close() + + async def test_intercepts_request_iterator_rpc_error_using_write(self): + for interceptor_class in (_StreamUnaryInterceptorEmpty, + _StreamUnaryInterceptorWithRequestIterator): + + with self.subTest(name=interceptor_class): + channel = aio.insecure_channel( + UNREACHABLE_TARGET, interceptors=[interceptor_class()]) + stub = test_pb2_grpc.TestServiceStub(channel) + + payload = messages_pb2.Payload(body=b'\0' * + _REQUEST_PAYLOAD_SIZE) + request = messages_pb2.StreamingInputCallRequest( + payload=payload) + + call = stub.StreamingInputCall() + + # When there is an error during the write, exception is raised. + with self.assertRaises(asyncio.InvalidStateError): + for _ in range(_NUM_STREAM_REQUESTS): + await call.write(request) + + with self.assertRaises(aio.AioRpcError) as exception_context: + await call + + self.assertEqual(grpc.StatusCode.UNAVAILABLE, + exception_context.exception.code()) + self.assertTrue(call.done()) + self.assertEqual(grpc.StatusCode.UNAVAILABLE, await call.code()) + + await channel.close() + + async def test_cancel_before_rpc(self): + + interceptor_reached = asyncio.Event() + wait_for_ever = self.loop.create_future() + + class Interceptor(aio.StreamUnaryClientInterceptor): + + async def intercept_stream_unary(self, continuation, + client_call_details, + request_iterator): + interceptor_reached.set() + await wait_for_ever + + channel = aio.insecure_channel(self._server_target, + interceptors=[Interceptor()]) + stub = test_pb2_grpc.TestServiceStub(channel) + + payload = messages_pb2.Payload(body=b'\0' * _REQUEST_PAYLOAD_SIZE) + request = messages_pb2.StreamingInputCallRequest(payload=payload) + + call = stub.StreamingInputCall() + + self.assertFalse(call.cancelled()) + self.assertFalse(call.done()) + + await interceptor_reached.wait() + self.assertTrue(call.cancel()) + + # When there is an error during the write, exception is raised. + with self.assertRaises(asyncio.InvalidStateError): + for _ in range(_NUM_STREAM_REQUESTS): + await call.write(request) + + with self.assertRaises(asyncio.CancelledError): + await call + + self.assertTrue(call.cancelled()) + self.assertTrue(call.done()) + self.assertEqual(await call.code(), grpc.StatusCode.CANCELLED) + self.assertEqual(await call.initial_metadata(), None) + self.assertEqual(await call.trailing_metadata(), None) + await channel.close() + + async def test_cancel_after_rpc(self): + + interceptor_reached = asyncio.Event() + wait_for_ever = self.loop.create_future() + + class Interceptor(aio.StreamUnaryClientInterceptor): + + async def intercept_stream_unary(self, continuation, + client_call_details, + request_iterator): + call = await continuation(client_call_details, request_iterator) + interceptor_reached.set() + await wait_for_ever + + channel = aio.insecure_channel(self._server_target, + interceptors=[Interceptor()]) + stub = test_pb2_grpc.TestServiceStub(channel) + + payload = messages_pb2.Payload(body=b'\0' * _REQUEST_PAYLOAD_SIZE) + request = messages_pb2.StreamingInputCallRequest(payload=payload) + + call = stub.StreamingInputCall() + + self.assertFalse(call.cancelled()) + self.assertFalse(call.done()) + + await interceptor_reached.wait() + self.assertTrue(call.cancel()) + + # When there is an error during the write, exception is raised. + with self.assertRaises(asyncio.InvalidStateError): + for _ in range(_NUM_STREAM_REQUESTS): + await call.write(request) + + with self.assertRaises(asyncio.CancelledError): + await call + + self.assertTrue(call.cancelled()) + self.assertTrue(call.done()) + self.assertEqual(await call.code(), grpc.StatusCode.CANCELLED) + self.assertEqual(await call.initial_metadata(), None) + self.assertEqual(await call.trailing_metadata(), None) + await channel.close() + + async def test_cancel_while_writing(self): + # Test cancelation before making any write or after doing at least 1 + for num_writes_before_cancel in (0, 1): + with self.subTest(name="Num writes before cancel: {}".format( + num_writes_before_cancel)): + + channel = aio.insecure_channel( + UNREACHABLE_TARGET, + interceptors=[_StreamUnaryInterceptorWithRequestIterator()]) + stub = test_pb2_grpc.TestServiceStub(channel) + + payload = messages_pb2.Payload(body=b'\0' * + _REQUEST_PAYLOAD_SIZE) + request = messages_pb2.StreamingInputCallRequest( + payload=payload) + + call = stub.StreamingInputCall() + + with self.assertRaises(asyncio.InvalidStateError): + for i in range(_NUM_STREAM_REQUESTS): + if i == num_writes_before_cancel: + self.assertTrue(call.cancel()) + await call.write(request) + + with self.assertRaises(asyncio.CancelledError): + await call + + self.assertTrue(call.cancelled()) + self.assertTrue(call.done()) + self.assertEqual(await call.code(), grpc.StatusCode.CANCELLED) + + await channel.close() + + async def test_cancel_by_the_interceptor(self): + + class Interceptor(aio.StreamUnaryClientInterceptor): + + async def intercept_stream_unary(self, continuation, + client_call_details, + request_iterator): + call = await continuation(client_call_details, request_iterator) + call.cancel() + return call + + channel = aio.insecure_channel(UNREACHABLE_TARGET, + interceptors=[Interceptor()]) + stub = test_pb2_grpc.TestServiceStub(channel) + + payload = messages_pb2.Payload(body=b'\0' * _REQUEST_PAYLOAD_SIZE) + request = messages_pb2.StreamingInputCallRequest(payload=payload) + + call = stub.StreamingInputCall() + + with self.assertRaises(asyncio.InvalidStateError): + for i in range(_NUM_STREAM_REQUESTS): + await call.write(request) + + with self.assertRaises(asyncio.CancelledError): + await call + + self.assertTrue(call.cancelled()) + self.assertTrue(call.done()) + self.assertEqual(await call.code(), grpc.StatusCode.CANCELLED) + + await channel.close() + + async def test_exception_raised_by_interceptor(self): + + class InterceptorException(Exception): + pass + + class Interceptor(aio.StreamUnaryClientInterceptor): + + async def intercept_stream_unary(self, continuation, + client_call_details, + request_iterator): + raise InterceptorException + + channel = aio.insecure_channel(UNREACHABLE_TARGET, + interceptors=[Interceptor()]) + stub = test_pb2_grpc.TestServiceStub(channel) + + payload = messages_pb2.Payload(body=b'\0' * _REQUEST_PAYLOAD_SIZE) + request = messages_pb2.StreamingInputCallRequest(payload=payload) + + call = stub.StreamingInputCall() + + with self.assertRaises(InterceptorException): + for i in range(_NUM_STREAM_REQUESTS): + await call.write(request) + + with self.assertRaises(InterceptorException): + await call + + await channel.close() + + async def test_intercepts_prohibit_mixing_style(self): + channel = aio.insecure_channel( + self._server_target, interceptors=[_StreamUnaryInterceptorEmpty()]) + stub = test_pb2_grpc.TestServiceStub(channel) + + payload = messages_pb2.Payload(body=b'\0' * _REQUEST_PAYLOAD_SIZE) + request = messages_pb2.StreamingInputCallRequest(payload=payload) + + async def request_iterator(): + for _ in range(_NUM_STREAM_REQUESTS): + yield request + + call = stub.StreamingInputCall(request_iterator()) + + with self.assertRaises(grpc._cython.cygrpc.UsageError): + await call.write(request) + + with self.assertRaises(grpc._cython.cygrpc.UsageError): + await call.done_writing() + + await channel.close() + + +if __name__ == '__main__': + logging.basicConfig(level=logging.DEBUG) + unittest.main(verbosity=2) From b9558026b1f7f15de8a27d5361404b1e61bb3f8b Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Wed, 29 Apr 2020 23:10:32 +0200 Subject: [PATCH 20/47] Improve message redability --- src/python/grpcio/grpc/experimental/aio/_interceptor.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/python/grpcio/grpc/experimental/aio/_interceptor.py b/src/python/grpcio/grpc/experimental/aio/_interceptor.py index 6a9f8434a0a..855f8aba708 100644 --- a/src/python/grpcio/grpc/experimental/aio/_interceptor.py +++ b/src/python/grpcio/grpc/experimental/aio/_interceptor.py @@ -579,9 +579,7 @@ class InterceptedStreamUnaryCall(_InterceptedUnaryResponseMixin, self._channel = channel if not request_iterator: # We provide our own request iterator which is a proxy - # of the future wries done by the caller. This iterator - # will use internally a queue for consuming messages produced - # by the write method. + # of the futures writes that will be done by the caller. self._write_to_iterator_queue = asyncio.Queue(maxsize=1) self._write_to_iterator_async_gen = self._proxies_writes_as_a_request_iteerator( ) From 7b869e8442111cf2fe5810459446431bd8d03766 Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Wed, 29 Apr 2020 23:12:02 +0200 Subject: [PATCH 21/47] Use message constant --- src/python/grpcio/grpc/experimental/aio/_interceptor.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/python/grpcio/grpc/experimental/aio/_interceptor.py b/src/python/grpcio/grpc/experimental/aio/_interceptor.py index 855f8aba708..eff5e03fc1e 100644 --- a/src/python/grpcio/grpc/experimental/aio/_interceptor.py +++ b/src/python/grpcio/grpc/experimental/aio/_interceptor.py @@ -650,8 +650,7 @@ class InterceptedStreamUnaryCall(_InterceptedUnaryResponseMixin, # should be expected through an iterators provided # by the caller. if self._write_to_iterator_queue is None: - raise cygrpc.UsageError( - 'Please don\'t mix two styles of API for streaming requests') + raise cygrpc.UsageError(_API_STYLE_ERROR) try: call = await self._interceptors_task From 2a523b03f36c943f95ec4851cc653b303d6a7fa9 Mon Sep 17 00:00:00 2001 From: Donna Dionne Date: Wed, 29 Apr 2020 14:53:05 -0700 Subject: [PATCH 22/47] Passing C repo manager to donnadionne --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- .github/ISSUE_TEMPLATE/cleanup_request.md | 2 +- .github/ISSUE_TEMPLATE/feature_request.md | 2 +- .github/pull_request_template.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 74060283ce2..680cd706703 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -2,7 +2,7 @@ name: Report a bug about: Create a report to help us improve labels: kind/bug, priority/P2 -assignees: veblush +assignees: donnadionne --- diff --git a/.github/ISSUE_TEMPLATE/cleanup_request.md b/.github/ISSUE_TEMPLATE/cleanup_request.md index 6e5c5aec48a..96c8bda3c4c 100644 --- a/.github/ISSUE_TEMPLATE/cleanup_request.md +++ b/.github/ISSUE_TEMPLATE/cleanup_request.md @@ -2,7 +2,7 @@ name: Request a cleanup about: Suggest a cleanup in our repository labels: kind/internal cleanup, priority/P2 -assignees: veblush +assignees: donnadionne --- diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index ba5d92da901..a7b05e99004 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -2,7 +2,7 @@ name: Request a feature about: Suggest an idea for this project labels: kind/enhancement, priority/P2 -assignees: veblush +assignees: donnadionne --- diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 366b68604df..eddba6676da 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -8,4 +8,4 @@ If you know who should review your pull request, please remove the mentioning be --> -@veblush +@donnadionne From 6cf73030a9f3cc498813f251a7916b59014fee19 Mon Sep 17 00:00:00 2001 From: ZhenLian Date: Wed, 29 Apr 2020 14:48:45 -0700 Subject: [PATCH 23/47] add comments to avoid future use of hard-coded creds --- test/core/end2end/data/ssl_test_data.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/core/end2end/data/ssl_test_data.h b/test/core/end2end/data/ssl_test_data.h index 303f3a6eda3..c679ffaf131 100644 --- a/test/core/end2end/data/ssl_test_data.h +++ b/test/core/end2end/data/ssl_test_data.h @@ -19,6 +19,10 @@ #ifndef GRPC_TEST_CORE_END2END_DATA_SSL_TEST_DATA_H #define GRPC_TEST_CORE_END2END_DATA_SSL_TEST_DATA_H +// These credentials are hardcoded as char arrays and are hence considered to +// be deprecated. Please consider using credentials in +// "src/core/tsi/test_creds" instead. + extern const char test_root_cert[]; extern const char test_server1_cert[]; extern const char test_server1_key[]; From df065d41fa958485c0d9207152356ce861f1dc64 Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Tue, 28 Apr 2020 17:55:13 -0700 Subject: [PATCH 24/47] Make sync handlers runnable in AsyncIO server --- .../grpc/_cython/_cygrpc/aio/common.pyx.pxi | 54 ++++++ .../grpc/_cython/_cygrpc/aio/server.pxd.pxi | 10 +- .../grpc/_cython/_cygrpc/aio/server.pyx.pxi | 171 ++++++++++++++--- .../tests_aio/unit/_test_server.py | 6 +- .../tests_aio/unit/compatibility_test.py | 179 +++++++++++++++++- 5 files changed, 385 insertions(+), 35 deletions(-) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/aio/common.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/aio/common.pyx.pxi index 6af2499e6e8..73b43815a1d 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/aio/common.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/aio/common.pyx.pxi @@ -112,3 +112,57 @@ def schedule_coro_threadsafe(object coro, object loop): ) else: raise + + +def async_generator_to_generator(object agen, object loop): + """Converts an async generator into generator.""" + try: + while True: + future = asyncio.run_coroutine_threadsafe( + agen.__anext__(), + loop + ) + response = future.result() + if response is EOF: + break + else: + yield response + except StopAsyncIteration: + # If StopAsyncIteration is raised, end this generator. + pass + + +async def generator_to_async_generator(object gen, object loop, object thread_pool): + """Converts a generator into async generator. + + The generator might block, so we need to delegate the iteration to thread + pool. Also, we can't simply delegate __next__ to the thread pool, otherwise + we will see following error: + + TypeError: StopIteration interacts badly with generators and cannot be + raised into a Future + """ + queue = asyncio.Queue(loop=loop) + + def yield_to_queue(): + try: + for item in gen: + # For an infinite sized queue, the put_nowait should always success + loop.call_soon_threadsafe(queue.put_nowait, item) + finally: + loop.call_soon_threadsafe(queue.put_nowait, EOF) + + future = loop.run_in_executor( + thread_pool, + yield_to_queue, + ) + + while True: + response = await queue.get() + if response is EOF: + break + else: + yield response + + # Port the exception if there is any + future.result() diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pxd.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pxd.pxi index 52f491614f1..46a47bd1ba7 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pxd.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pxd.pxi @@ -48,6 +48,12 @@ cdef class _ServicerContext: cdef object _response_serializer # Callable[[Any], bytes] +cdef class _SyncServicerContext: + cdef _ServicerContext _context + cdef list _callbacks + cdef object _loop # asyncio.AbstractEventLoop + + cdef class _MessageReceiver: cdef _ServicerContext _servicer_context cdef object _agen @@ -71,5 +77,7 @@ cdef class AioServer: cdef object _shutdown_completed # asyncio.Future cdef CallbackWrapper _shutdown_callback_wrapper cdef object _crash_exception # Exception - cdef set _ongoing_rpc_tasks cdef tuple _interceptors + cdef object _thread_pool # concurrent.futures.ThreadPoolExecutor + + cdef thread_pool(self) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi index 0e407172806..f015bce5c3b 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi @@ -211,6 +211,65 @@ cdef class _ServicerContext: self._rpc_state.disable_next_compression = True +cdef class _SyncServicerContext: + """Sync servicer context for sync handler compatibility.""" + + def __cinit__(self, + _ServicerContext context): + self._context = context + self._callbacks = [] + self._loop = context._loop + + def read(self): + future = asyncio.run_coroutine_threadsafe( + self._context.read(), + self._loop) + return future.result() + + def write(self, object message): + future = asyncio.run_coroutine_threadsafe( + self._context.write(message), + self._loop) + future.result() + + def abort(self, + object code, + str details='', + tuple trailing_metadata=_IMMUTABLE_EMPTY_METADATA): + future = asyncio.run_coroutine_threadsafe( + self._context.abort(code, details, trailing_metadata), + self._loop) + # Abort should raise an AbortError + future.exception() + + def send_initial_metadata(self, tuple metadata): + future = asyncio.run_coroutine_threadsafe( + self._context.send_initial_metadata(metadata), + self._loop) + future.result() + + def set_trailing_metadata(self, tuple metadata): + self._context.set_trailing_metadata(metadata) + + def invocation_metadata(self): + return self._context.invocation_metadata() + + def set_code(self, object code): + self._context.set_code(code) + + def set_details(self, str details): + self._context.set_details(details) + + def set_compression(self, object compression): + self._context.set_compression(compression) + + def disable_next_message_compression(self): + self._context.disable_next_message_compression() + + def add_callback(self, object callback): + self._callbacks.append(callback) + + async def _run_interceptor(object interceptors, object query_handler, object handler_call_details): interceptor = next(interceptors, None) @@ -222,6 +281,11 @@ async def _run_interceptor(object interceptors, object query_handler, return query_handler(handler_call_details) +def _is_async_handler(object handler): + """Inspect if a method handler is async or sync.""" + return inspect.isawaitable(handler) or inspect.iscoroutinefunction(handler) or inspect.isasyncgenfunction(handler) + + async def _find_method_handler(str method, tuple metadata, list generic_handlers, tuple interceptors): def query_handlers(handler_call_details): @@ -254,11 +318,27 @@ async def _finish_handler_with_unary_response(RPCState rpc_state, stream-unary handlers. """ # Executes application logic - - cdef object response_message = await unary_handler( - request, - servicer_context, - ) + cdef object response_message + cdef _SyncServicerContext sync_servicer_context + + if _is_async_handler(unary_handler): + # Run async method handlers in this coroutine + response_message = await unary_handler( + request, + servicer_context, + ) + else: + # Run sync method handlers in the thread pool + sync_servicer_context = _SyncServicerContext(servicer_context) + response_message = await loop.run_in_executor( + rpc_state.server.thread_pool(), + unary_handler, + request, + sync_servicer_context, + ) + # Support sync-stack callback + for callback in sync_servicer_context._callbacks: + callback() # Raises exception if aborted rpc_state.raise_for_termination() @@ -307,18 +387,31 @@ async def _finish_handler_with_stream_responses(RPCState rpc_state, """ cdef object async_response_generator cdef object response_message + if inspect.iscoroutinefunction(stream_handler): + # Case 1: Coroutine async handler - using reader-writer API # The handler uses reader / writer API, returns None. await stream_handler( request, servicer_context, ) else: - # The handler uses async generator API - async_response_generator = stream_handler( - request, - servicer_context, - ) + if inspect.isasyncgenfunction(stream_handler): + # Case 2: Async handler - async generator + # The handler uses async generator API + async_response_generator = stream_handler( + request, + servicer_context, + ) + else: + # Case 3: Sync handler - normal generator + # NOTE(lidiz) Streaming handler in sync stack is either a generator + # function or a function returns a generator. + sync_servicer_context = _SyncServicerContext(servicer_context) + gen = stream_handler(request, sync_servicer_context) + async_response_generator = generator_to_async_generator(gen, + loop, + rpc_state.server.thread_pool()) # Consumes messages from the generator async for response_message in async_response_generator: @@ -438,6 +531,9 @@ cdef class _MessageReceiver: self._agen = self._async_message_receiver() return self._agen + async def __anext__(self): + return await self.__aiter__().__anext__() + async def _handle_stream_unary_rpc(object method_handler, RPCState rpc_state, @@ -451,13 +547,20 @@ async def _handle_stream_unary_rpc(object method_handler, ) # Prepares the request generator - cdef object request_async_iterator = _MessageReceiver(servicer_context) + cdef object request_iterator + if _is_async_handler(method_handler.stream_unary): + request_iterator = _MessageReceiver(servicer_context) + else: + request_iterator = async_generator_to_generator( + _MessageReceiver(servicer_context), + loop + ) # Finishes the application handler await _finish_handler_with_unary_response( rpc_state, method_handler.stream_unary, - request_async_iterator, + request_iterator, servicer_context, method_handler.response_serializer, loop @@ -476,13 +579,20 @@ async def _handle_stream_stream_rpc(object method_handler, ) # Prepares the request generator - cdef object request_async_iterator = _MessageReceiver(servicer_context) + cdef object request_iterator + if _is_async_handler(method_handler.stream_stream): + request_iterator = _MessageReceiver(servicer_context) + else: + request_iterator = async_generator_to_generator( + _MessageReceiver(servicer_context), + loop + ) # Finishes the application handler await _finish_handler_with_stream_responses( rpc_state, method_handler.stream_stream, - request_async_iterator, + request_iterator, servicer_context, loop, ) @@ -591,22 +701,22 @@ async def _handle_rpc(list generic_handlers, tuple interceptors, # Handles unary-unary case if not method_handler.request_streaming and not method_handler.response_streaming: await _handle_unary_unary_rpc(method_handler, - rpc_state, - loop) + rpc_state, + loop) return # Handles unary-stream case if not method_handler.request_streaming and method_handler.response_streaming: await _handle_unary_stream_rpc(method_handler, - rpc_state, - loop) + rpc_state, + loop) return # Handles stream-unary case if method_handler.request_streaming and not method_handler.response_streaming: await _handle_stream_unary_rpc(method_handler, - rpc_state, - loop) + rpc_state, + loop) return # Handles stream-stream case @@ -648,7 +758,6 @@ cdef class AioServer: self._generic_handlers = [] self.add_generic_rpc_handlers(generic_handlers) self._serving_task = None - self._ongoing_rpc_tasks = set() self._shutdown_lock = asyncio.Lock(loop=self._loop) self._shutdown_completed = self._loop.create_future() @@ -658,17 +767,18 @@ cdef class AioServer: SERVER_SHUTDOWN_FAILURE_HANDLER) self._crash_exception = None - self._interceptors = () if interceptors: self._interceptors = interceptors + else: + self._interceptors = () + + self._thread_pool = thread_pool + if maximum_concurrent_rpcs: raise NotImplementedError() - if thread_pool: - raise NotImplementedError() - def add_generic_rpc_handlers(self, generic_rpc_handlers): - for h in generic_rpc_handlers: - self._generic_handlers.append(h) + def add_generic_rpc_handlers(self, object generic_rpc_handlers): + self._generic_handlers.extend(generic_rpc_handlers) def add_insecure_port(self, address): return self._server.add_http2_port(address) @@ -846,3 +956,10 @@ cdef class AioServer: self._status ) shutdown_grpc_aio() + + cdef thread_pool(self): + """Access the thread pool instance.""" + if self._thread_pool: + return self._thread_pool + else: + raise UsageError('Please provide an Executor upon server creation.') diff --git a/src/python/grpcio_tests/tests_aio/unit/_test_server.py b/src/python/grpcio_tests/tests_aio/unit/_test_server.py index 2396608e5cc..5e5081a38d0 100644 --- a/src/python/grpcio_tests/tests_aio/unit/_test_server.py +++ b/src/python/grpcio_tests/tests_aio/unit/_test_server.py @@ -47,7 +47,7 @@ async def _maybe_echo_status(request: messages_pb2.SimpleRequest, request.response_status.message) -class _TestServiceServicer(test_pb2_grpc.TestServiceServicer): +class TestServiceServicer(test_pb2_grpc.TestServiceServicer): async def UnaryCall(self, request, context): await _maybe_echo_metadata(context) @@ -102,7 +102,7 @@ class _TestServiceServicer(test_pb2_grpc.TestServiceServicer): response_parameters.size)) -def _create_extra_generic_handler(servicer: _TestServiceServicer): +def _create_extra_generic_handler(servicer: TestServiceServicer): # Add programatically extra methods not provided by the proto file # that are used during the tests rpc_method_handlers = { @@ -123,7 +123,7 @@ async def start_test_server(port=0, interceptors=None): server = aio.server(options=(('grpc.so_reuseport', 0),), interceptors=interceptors) - servicer = _TestServiceServicer() + servicer = TestServiceServicer() test_pb2_grpc.add_TestServiceServicer_to_server(servicer, server) server.add_generic_rpc_handlers((_create_extra_generic_handler(servicer),)) diff --git a/src/python/grpcio_tests/tests_aio/unit/compatibility_test.py b/src/python/grpcio_tests/tests_aio/unit/compatibility_test.py index e3d85e70e69..fac39c427ec 100644 --- a/src/python/grpcio_tests/tests_aio/unit/compatibility_test.py +++ b/src/python/grpcio_tests/tests_aio/unit/compatibility_test.py @@ -20,32 +20,63 @@ import random import threading import unittest from concurrent.futures import ThreadPoolExecutor -from typing import Callable, Sequence, Tuple +from typing import Callable, Iterable, Sequence, Tuple import grpc from grpc.experimental import aio from src.proto.grpc.testing import messages_pb2, test_pb2_grpc from tests.unit.framework.common import test_constants +from tests_aio.unit import _common from tests_aio.unit._test_base import AioTestBase -from tests_aio.unit._test_server import start_test_server +from tests_aio.unit._test_server import TestServiceServicer, start_test_server _NUM_STREAM_RESPONSES = 5 _REQUEST_PAYLOAD_SIZE = 7 _RESPONSE_PAYLOAD_SIZE = 42 +_REQUEST = b'\x03\x07' +_ADHOC_METHOD = '/test/AdHoc' def _unique_options() -> Sequence[Tuple[str, float]]: return (('iv', random.random()),) +class _AdhocGenericHandler(grpc.GenericRpcHandler): + _handler: grpc.RpcMethodHandler + + def __init__(self): + self._handler = None + + def set_adhoc_handler(self, handler: grpc.RpcMethodHandler): + self._handler = handler + + def service(self, handler_call_details: grpc.HandlerCallDetails): + if handler_call_details.method == _ADHOC_METHOD: + return self._handler + else: + return None + + @unittest.skipIf( - os.environ.get('GRPC_ASYNCIO_ENGINE', '').lower() != 'poller', + os.environ.get('GRPC_ASYNCIO_ENGINE', '').lower() == 'custom_io_manager', 'Compatible mode needs POLLER completion queue.') class TestCompatibility(AioTestBase): async def setUp(self): - address, self._async_server = await start_test_server() + self._async_server = aio.server( + options=(('grpc.so_reuseport', 0),), + migration_thread_pool=ThreadPoolExecutor()) + + test_pb2_grpc.add_TestServiceServicer_to_server(TestServiceServicer(), + self._async_server) + self._adhoc_handlers = _AdhocGenericHandler() + self._async_server.add_generic_rpc_handlers((self._adhoc_handlers,)) + + port = self._async_server.add_insecure_port('[::]:0') + address = 'localhost:%d' % port + await self._async_server.start() + # Create async stub self._async_channel = aio.insecure_channel(address, options=_unique_options()) @@ -202,6 +233,146 @@ class TestCompatibility(AioTestBase): await self._run_in_another_thread(sync_work) await server.stop(None) + async def test_sync_unary_unary_success(self): + + @grpc.unary_unary_rpc_method_handler + def echo_unary_unary(request: bytes, unused_context): + return request + + self._adhoc_handlers.set_adhoc_handler(echo_unary_unary) + response = await self._async_channel.unary_unary(_ADHOC_METHOD)(_REQUEST + ) + self.assertEqual(_REQUEST, response) + + async def test_sync_unary_unary_metadata(self): + metadata = (('unique', 'key-42'),) + + @grpc.unary_unary_rpc_method_handler + def metadata_unary_unary(request: bytes, context: grpc.ServicerContext): + context.send_initial_metadata(metadata) + return request + + self._adhoc_handlers.set_adhoc_handler(metadata_unary_unary) + call = self._async_channel.unary_unary(_ADHOC_METHOD)(_REQUEST) + self.assertTrue( + _common.seen_metadata(metadata, await call.initial_metadata())) + + async def test_sync_unary_unary_abort(self): + + @grpc.unary_unary_rpc_method_handler + def abort_unary_unary(request: bytes, context: grpc.ServicerContext): + context.abort(grpc.StatusCode.INTERNAL, 'Test') + + self._adhoc_handlers.set_adhoc_handler(abort_unary_unary) + with self.assertRaises(aio.AioRpcError) as exception_context: + await self._async_channel.unary_unary(_ADHOC_METHOD)(_REQUEST) + self.assertEqual(grpc.StatusCode.INTERNAL, + exception_context.exception.code()) + + async def test_sync_unary_unary_set_code(self): + + @grpc.unary_unary_rpc_method_handler + def set_code_unary_unary(request: bytes, context: grpc.ServicerContext): + context.set_code(grpc.StatusCode.INTERNAL) + + self._adhoc_handlers.set_adhoc_handler(set_code_unary_unary) + with self.assertRaises(aio.AioRpcError) as exception_context: + await self._async_channel.unary_unary(_ADHOC_METHOD)(_REQUEST) + self.assertEqual(grpc.StatusCode.INTERNAL, + exception_context.exception.code()) + + async def test_sync_unary_stream_success(self): + + @grpc.unary_stream_rpc_method_handler + def echo_unary_stream(request: bytes, unused_context): + for _ in range(_NUM_STREAM_RESPONSES): + yield request + + self._adhoc_handlers.set_adhoc_handler(echo_unary_stream) + call = self._async_channel.unary_stream(_ADHOC_METHOD)(_REQUEST) + async for response in call: + self.assertEqual(_REQUEST, response) + + async def test_sync_unary_stream_error(self): + + @grpc.unary_stream_rpc_method_handler + def error_unary_stream(request: bytes, unused_context): + for _ in range(_NUM_STREAM_RESPONSES): + yield request + raise RuntimeError('Test') + + self._adhoc_handlers.set_adhoc_handler(error_unary_stream) + call = self._async_channel.unary_stream(_ADHOC_METHOD)(_REQUEST) + with self.assertRaises(aio.AioRpcError) as exception_context: + async for response in call: + self.assertEqual(_REQUEST, response) + self.assertEqual(grpc.StatusCode.UNKNOWN, + exception_context.exception.code()) + + async def test_sync_stream_unary_success(self): + + @grpc.stream_unary_rpc_method_handler + def echo_stream_unary(request_iterator: Iterable[bytes], + unused_context): + self.assertEqual(len(list(request_iterator)), _NUM_STREAM_RESPONSES) + return _REQUEST + + self._adhoc_handlers.set_adhoc_handler(echo_stream_unary) + request_iterator = iter([_REQUEST] * _NUM_STREAM_RESPONSES) + response = await self._async_channel.stream_unary(_ADHOC_METHOD)( + request_iterator) + self.assertEqual(_REQUEST, response) + + async def test_sync_stream_unary_error(self): + + @grpc.stream_unary_rpc_method_handler + def echo_stream_unary(request_iterator: Iterable[bytes], + unused_context): + self.assertEqual(len(list(request_iterator)), _NUM_STREAM_RESPONSES) + raise RuntimeError('Test') + + self._adhoc_handlers.set_adhoc_handler(echo_stream_unary) + request_iterator = iter([_REQUEST] * _NUM_STREAM_RESPONSES) + with self.assertRaises(aio.AioRpcError) as exception_context: + response = await self._async_channel.stream_unary(_ADHOC_METHOD)( + request_iterator) + self.assertEqual(grpc.StatusCode.UNKNOWN, + exception_context.exception.code()) + + async def test_sync_stream_stream_success(self): + + @grpc.stream_stream_rpc_method_handler + def echo_stream_stream(request_iterator: Iterable[bytes], + unused_context): + for request in request_iterator: + yield request + + self._adhoc_handlers.set_adhoc_handler(echo_stream_stream) + request_iterator = iter([_REQUEST] * _NUM_STREAM_RESPONSES) + call = self._async_channel.stream_stream(_ADHOC_METHOD)( + request_iterator) + async for response in call: + self.assertEqual(_REQUEST, response) + + async def test_sync_stream_stream_error(self): + + @grpc.stream_stream_rpc_method_handler + def echo_stream_stream(request_iterator: Iterable[bytes], + unused_context): + for request in request_iterator: + yield request + raise RuntimeError('test') + + self._adhoc_handlers.set_adhoc_handler(echo_stream_stream) + request_iterator = iter([_REQUEST] * _NUM_STREAM_RESPONSES) + call = self._async_channel.stream_stream(_ADHOC_METHOD)( + request_iterator) + with self.assertRaises(aio.AioRpcError) as exception_context: + async for response in call: + self.assertEqual(_REQUEST, response) + self.assertEqual(grpc.StatusCode.UNKNOWN, + exception_context.exception.code()) + if __name__ == '__main__': logging.basicConfig(level=logging.DEBUG) From d40c1b84081d28859cd2e9402e8c3ec5b1510dad Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Wed, 29 Apr 2020 11:40:31 -0700 Subject: [PATCH 25/47] Fix the error on mac and win && make pytype happy --- src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi | 3 ++- src/python/grpcio_tests/tests_aio/unit/compatibility_test.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi index f015bce5c3b..b92d681aba0 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi @@ -962,4 +962,5 @@ cdef class AioServer: if self._thread_pool: return self._thread_pool else: - raise UsageError('Please provide an Executor upon server creation.') + # Use the event loop's default executor + return None diff --git a/src/python/grpcio_tests/tests_aio/unit/compatibility_test.py b/src/python/grpcio_tests/tests_aio/unit/compatibility_test.py index fac39c427ec..1e6e3598b8b 100644 --- a/src/python/grpcio_tests/tests_aio/unit/compatibility_test.py +++ b/src/python/grpcio_tests/tests_aio/unit/compatibility_test.py @@ -51,7 +51,7 @@ class _AdhocGenericHandler(grpc.GenericRpcHandler): def set_adhoc_handler(self, handler: grpc.RpcMethodHandler): self._handler = handler - def service(self, handler_call_details: grpc.HandlerCallDetails): + def service(self, handler_call_details): if handler_call_details.method == _ADHOC_METHOD: return self._handler else: From 8be1251e2c5e15b0439fcdcc6b00dd2232e43d3c Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Wed, 29 Apr 2020 15:37:15 -0700 Subject: [PATCH 26/47] Remove read/write API && simplify the logic of getting the pool --- .../grpc/_cython/_cygrpc/aio/server.pyx.pxi | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi index b92d681aba0..7ca16e23198 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi @@ -220,18 +220,6 @@ cdef class _SyncServicerContext: self._callbacks = [] self._loop = context._loop - def read(self): - future = asyncio.run_coroutine_threadsafe( - self._context.read(), - self._loop) - return future.result() - - def write(self, object message): - future = asyncio.run_coroutine_threadsafe( - self._context.write(message), - self._loop) - future.result() - def abort(self, object code, str details='', @@ -959,8 +947,4 @@ cdef class AioServer: cdef thread_pool(self): """Access the thread pool instance.""" - if self._thread_pool: - return self._thread_pool - else: - # Use the event loop's default executor - return None + return self._thread_pool From 469631f54965ac972794fd10a55159287298f629 Mon Sep 17 00:00:00 2001 From: Taras Galkovskyi Date: Wed, 29 Apr 2020 22:50:35 +0000 Subject: [PATCH 27/47] run yapf; delete alts unit tests --- .../python/data_transmission/alts_client.py | 4 +- .../python/data_transmission/alts_server.py | 3 +- src/python/grpcio/grpc/__init__.py | 5 +- .../grpc/_cython/_cygrpc/credentials.pyx.pxi | 1 + .../grpcio_tests/tests/interop/server.py | 2 - src/python/grpcio_tests/tests/tests.json | 1 - .../grpcio_tests/tests/unit/BUILD.bazel | 1 - .../tests/unit/_alts_credentials_test.py | 57 ------------------- 8 files changed, 9 insertions(+), 65 deletions(-) delete mode 100644 src/python/grpcio_tests/tests/unit/_alts_credentials_test.py diff --git a/examples/python/data_transmission/alts_client.py b/examples/python/data_transmission/alts_client.py index a2ff05e3646..d196dbd17c2 100644 --- a/examples/python/data_transmission/alts_client.py +++ b/examples/python/data_transmission/alts_client.py @@ -22,9 +22,11 @@ import demo_pb2_grpc SERVER_ADDRESS = "localhost:23333" + def main(): with grpc.secure_channel( - SERVER_ADDRESS, credentials=grpc.alts_channel_credentials()) as channel: + SERVER_ADDRESS, + credentials=grpc.alts_channel_credentials()) as channel: stub = demo_pb2_grpc.GRPCDemoStub(channel) client.simple_method(stub) client.client_streaming_method(stub) diff --git a/examples/python/data_transmission/alts_server.py b/examples/python/data_transmission/alts_server.py index 4f7b84d9ca6..9189405c916 100644 --- a/examples/python/data_transmission/alts_server.py +++ b/examples/python/data_transmission/alts_server.py @@ -27,7 +27,8 @@ SERVER_ADDRESS = 'localhost:23333' def main(): svr = grpc.server(futures.ThreadPoolExecutor()) demo_pb2_grpc.add_GRPCDemoServicer_to_server(server.DemoServer(), svr) - svr.add_secure_port(SERVER_ADDRESS, server_credentials=grpc.alts_server_credentials()) + svr.add_secure_port(SERVER_ADDRESS, + server_credentials=grpc.alts_server_credentials()) print("------------------start Python GRPC server with ALTS encryption") svr.start() svr.wait_for_termination() diff --git a/src/python/grpcio/grpc/__init__.py b/src/python/grpcio/grpc/__init__.py index 717c1734453..258fb5cf52b 100644 --- a/src/python/grpcio/grpc/__init__.py +++ b/src/python/grpcio/grpc/__init__.py @@ -1832,7 +1832,7 @@ def local_server_credentials(local_connect_type=LocalConnectionType.LOCAL_TCP): return ServerCredentials( _cygrpc.server_credentials_local(local_connect_type.value)) - + def alts_channel_credentials(service_accounts=[]): """Creates a ChannelCredentials for use with an ALTS-enabled Channel. @@ -1850,7 +1850,8 @@ def alts_channel_credentials(service_accounts=[]): Returns: A ChannelCredentials for use with an ALTS-enabled Channel """ - return ChannelCredentials(_cygrpc.channel_credentials_alts(service_accounts)) + return ChannelCredentials( + _cygrpc.channel_credentials_alts(service_accounts)) def alts_server_credentials(): diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi index d3d0f758b16..0ff3706d134 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi @@ -357,6 +357,7 @@ cdef class ALTSChannelCredentials(ChannelCredentials): def __cinit__(self, list service_accounts): self.c_options = grpc_alts_credentials_client_options_create() + cdef str account for account in service_accounts: grpc_alts_credentials_client_options_add_target_service_account(self.c_options, account) diff --git a/src/python/grpcio_tests/tests/interop/server.py b/src/python/grpcio_tests/tests/interop/server.py index a72294aeac8..c85adb0b0bb 100644 --- a/src/python/grpcio_tests/tests/interop/server.py +++ b/src/python/grpcio_tests/tests/interop/server.py @@ -54,8 +54,6 @@ def get_server_credentials(use_tls): return grpc.alts_server_credentials() - - def serve(): args = parse_interop_server_arguments() diff --git a/src/python/grpcio_tests/tests/tests.json b/src/python/grpcio_tests/tests/tests.json index 95fc8c77cf0..196e9f08b0a 100644 --- a/src/python/grpcio_tests/tests/tests.json +++ b/src/python/grpcio_tests/tests/tests.json @@ -24,7 +24,6 @@ "testing._time_test.StrictFakeTimeTest", "testing._time_test.StrictRealTimeTest", "unit._abort_test.AbortTest", - "unit._alts_credentials_test.ALTSCredentialsTest", "unit._api_test.AllTest", "unit._api_test.ChannelConnectivityTest", "unit._api_test.ChannelTest", diff --git a/src/python/grpcio_tests/tests/unit/BUILD.bazel b/src/python/grpcio_tests/tests/unit/BUILD.bazel index ec2e93538e2..42b99023463 100644 --- a/src/python/grpcio_tests/tests/unit/BUILD.bazel +++ b/src/python/grpcio_tests/tests/unit/BUILD.bazel @@ -4,7 +4,6 @@ package(default_visibility = ["//visibility:public"]) GRPCIO_TESTS_UNIT = [ "_abort_test.py", - "_alts_credentials_test.py", "_api_test.py", "_auth_context_test.py", "_auth_test.py", diff --git a/src/python/grpcio_tests/tests/unit/_alts_credentials_test.py b/src/python/grpcio_tests/tests/unit/_alts_credentials_test.py deleted file mode 100644 index 07a3c849c8b..00000000000 --- a/src/python/grpcio_tests/tests/unit/_alts_credentials_test.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2020 The gRPC Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Test of RPCs made using ALTS credentials.""" - -import unittest -import os -from concurrent.futures import ThreadPoolExecutor -import grpc - - -REQUEST = b'abc' - - -class _GenericHandler(grpc.GenericRpcHandler): - - def service(self, handler_call_details): - return grpc.unary_unary_rpc_method_handler( - lambda request, unused_context: request) - - -class ALTSCredentialsTest(unittest.TestCase): - - def _create_server(self): - server = grpc.server(ThreadPoolExecutor()) - server.add_generic_rpc_handlers((_GenericHandler(),)) - return server - - def test_alts(self): - server_addr = 'localhost:{}' - channel_creds = grpc.alts_channel_credentials([]) - server_creds = grpc.alts_server_credentials() - - server = self._create_server() - port = server.add_secure_port(server_addr.format(0), server_creds) - server.start() - with grpc.secure_channel(server_addr.format(port), - channel_creds) as channel: - self.assertEqual( - REQUEST, - channel.unary_unary('/test/method')(REQUEST, - wait_for_ready=True)) - server.stop(None) - - -if __name__ == '__main__': - unittest.main() From fb60bf235e66ce11c72ba27456b2773ef8c5cf49 Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Wed, 29 Apr 2020 16:11:55 -0700 Subject: [PATCH 28/47] Synchonize the production and consumption of request messages --- .../grpcio/grpc/_cython/_cygrpc/aio/common.pyx.pxi | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/aio/common.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/aio/common.pyx.pxi index 73b43815a1d..1798bef72f5 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/aio/common.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/aio/common.pyx.pxi @@ -142,15 +142,14 @@ async def generator_to_async_generator(object gen, object loop, object thread_po TypeError: StopIteration interacts badly with generators and cannot be raised into a Future """ - queue = asyncio.Queue(loop=loop) + queue = asyncio.Queue(maxsize=1, loop=loop) def yield_to_queue(): try: for item in gen: - # For an infinite sized queue, the put_nowait should always success - loop.call_soon_threadsafe(queue.put_nowait, item) + asyncio.run_coroutine_threadsafe(queue.put(item), loop).result() finally: - loop.call_soon_threadsafe(queue.put_nowait, EOF) + asyncio.run_coroutine_threadsafe(queue.put(EOF), loop).result() future = loop.run_in_executor( thread_pool, @@ -165,4 +164,4 @@ async def generator_to_async_generator(object gen, object loop, object thread_po yield response # Port the exception if there is any - future.result() + await future From d62df731a0db57185d378397a6fe21c63af1bd00 Mon Sep 17 00:00:00 2001 From: Taras Galkovskyi Date: Wed, 29 Apr 2020 23:22:22 +0000 Subject: [PATCH 29/47] pacify pylint on examples folder --- examples/python/data_transmission/client.py | 5 +++++ examples/python/data_transmission/server.py | 1 + 2 files changed, 6 insertions(+) diff --git a/examples/python/data_transmission/client.py b/examples/python/data_transmission/client.py index 262e366665f..cc4a68e3435 100644 --- a/examples/python/data_transmission/client.py +++ b/examples/python/data_transmission/client.py @@ -19,6 +19,11 @@ import grpc import demo_pb2_grpc import demo_pb2 +__all__ = [ + 'simple_method', 'client_streaming_method', 'server_streaming_method', + 'bidirectional_streaming_method' +] + SERVER_ADDRESS = "localhost:23333" CLIENT_ID = 1 diff --git a/examples/python/data_transmission/server.py b/examples/python/data_transmission/server.py index 0a659281261..3c990945ece 100644 --- a/examples/python/data_transmission/server.py +++ b/examples/python/data_transmission/server.py @@ -20,6 +20,7 @@ import grpc import demo_pb2_grpc import demo_pb2 +__all__ = 'DemoServer' SERVER_ADDRESS = 'localhost:23333' SERVER_ID = 1 From bad263d533cb3957b0d5c2066bac18878e8a2cbf Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Wed, 29 Apr 2020 16:04:37 -0700 Subject: [PATCH 30/47] update ruby gem email to grpc-io@googlegroups.com --- grpc.gemspec | 2 +- templates/grpc.gemspec.template | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/grpc.gemspec b/grpc.gemspec index a1f538c1501..77e05b656f5 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -7,7 +7,7 @@ Gem::Specification.new do |s| s.name = 'grpc' s.version = GRPC::VERSION s.authors = ['gRPC Authors'] - s.email = 'temiola@google.com' + s.email = 'grpc-io@googlegroups.com' s.homepage = 'https://github.com/google/grpc/tree/master/src/ruby' s.summary = 'GRPC system in Ruby' s.description = 'Send RPCs from Ruby using GRPC' diff --git a/templates/grpc.gemspec.template b/templates/grpc.gemspec.template index ba6ea179c5c..6a2f812f751 100644 --- a/templates/grpc.gemspec.template +++ b/templates/grpc.gemspec.template @@ -9,7 +9,7 @@ s.name = 'grpc' s.version = GRPC::VERSION s.authors = ['gRPC Authors'] - s.email = 'temiola@google.com' + s.email = 'grpc-io@googlegroups.com' s.homepage = 'https://github.com/google/grpc/tree/master/src/ruby' s.summary = 'GRPC system in Ruby' s.description = 'Send RPCs from Ruby using GRPC' From 121a59ba4d6fc474ab4db14c71bb03c65eee81f8 Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Wed, 29 Apr 2020 20:38:13 -0700 Subject: [PATCH 31/47] Disable strict --fail_on_failed_rpc flag generally --- tools/run_tests/run_xds_tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/run_tests/run_xds_tests.py b/tools/run_tests/run_xds_tests.py index ef3b0e91294..9b47b28d1ba 100755 --- a/tools/run_tests/run_xds_tests.py +++ b/tools/run_tests/run_xds_tests.py @@ -226,9 +226,9 @@ _BOOTSTRAP_TEMPLATE = """ # TODO(ericgribkoff) Add change_backend_service to this list once TD no longer # sends an update with no localities when adding the MIG to the backend service # can race with the URL map patch. -_TESTS_TO_FAIL_ON_RPC_FAILURE = [ - 'new_instance_group_receives_traffic', 'ping_pong', 'round_robin' -] +# TODO(ericgribkoff) Add new_instance_group_receives_traffic, ping_pong, and +# round_robin when empty update issue is resolved. +_TESTS_TO_FAIL_ON_RPC_FAILURE = [] _TESTS_USING_SECONDARY_IG = [ 'secondary_locality_gets_no_requests_on_partial_primary_failure', 'secondary_locality_gets_requests_on_primary_failure' From 791729d09ffb227d739fdb17c89502fc14b5870a Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Thu, 30 Apr 2020 08:12:29 +0200 Subject: [PATCH 32/47] Bumping version. --- BUILD | 4 ++-- CMakeLists.txt | 6 +++--- Makefile | 4 ++-- build_handwritten.yaml | 4 ++-- doc/g_stands_for.md | 3 ++- gRPC-C++.podspec | 2 +- gRPC-Core.podspec | 2 +- gRPC-ProtoRPC.podspec | 2 +- gRPC-RxLibrary.podspec | 2 +- gRPC.podspec | 2 +- package.xml | 6 +++--- src/core/lib/surface/version.cc | 2 +- src/cpp/common/version_cc.cc | 2 +- src/csharp/Grpc.Core.Api/VersionInfo.cs | 4 ++-- src/csharp/build/dependencies.props | 2 +- src/csharp/build_unitypackage.bat | 2 +- src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec | 2 +- src/objective-c/!ProtoCompiler-gRPCPlugin.podspec | 2 +- src/objective-c/GRPCClient/version.h | 2 +- src/objective-c/tests/version.h | 2 +- src/php/composer.json | 2 +- src/php/ext/grpc/version.h | 2 +- src/python/grpcio/grpc/_grpcio_metadata.py | 2 +- src/python/grpcio/grpc_version.py | 2 +- src/python/grpcio_channelz/grpc_version.py | 2 +- src/python/grpcio_health_checking/grpc_version.py | 2 +- src/python/grpcio_reflection/grpc_version.py | 2 +- src/python/grpcio_status/grpc_version.py | 2 +- src/python/grpcio_testing/grpc_version.py | 2 +- src/python/grpcio_tests/grpc_version.py | 2 +- src/ruby/lib/grpc/version.rb | 2 +- src/ruby/tools/version.rb | 2 +- tools/distrib/python/grpcio_tools/grpc_version.py | 2 +- tools/doxygen/Doxyfile.c++ | 2 +- tools/doxygen/Doxyfile.c++.internal | 2 +- tools/doxygen/Doxyfile.objc | 2 +- tools/doxygen/Doxyfile.objc.internal | 2 +- 37 files changed, 46 insertions(+), 45 deletions(-) diff --git a/BUILD b/BUILD index edc09c315ff..0eb776e785b 100644 --- a/BUILD +++ b/BUILD @@ -75,11 +75,11 @@ config_setting( python_config_settings() # This should be updated along with build.yaml -g_stands_for = "gringotts" +g_stands_for = "gradius" core_version = "10.0.0" -version = "1.29.0-dev" +version = "1.30.0-dev" GPR_PUBLIC_HDRS = [ "include/grpc/support/alloc.h", diff --git a/CMakeLists.txt b/CMakeLists.txt index 32f328341d7..7f18b7f77f5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,12 +25,12 @@ cmake_minimum_required(VERSION 3.5.1) set(PACKAGE_NAME "grpc") -set(PACKAGE_VERSION "1.29.0-dev") +set(PACKAGE_VERSION "1.30.0-dev") set(gRPC_CORE_VERSION "10.0.0") set(gRPC_CORE_SOVERSION "10") -set(gRPC_CPP_VERSION "1.29.0-dev") +set(gRPC_CPP_VERSION "1.30.0-dev") set(gRPC_CPP_SOVERSION "1") -set(gRPC_CSHARP_VERSION "2.29.0-dev") +set(gRPC_CSHARP_VERSION "2.30.0-dev") set(gRPC_CSHARP_SOVERSION "2") set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}") set(PACKAGE_TARNAME "${PACKAGE_NAME}-${PACKAGE_VERSION}") diff --git a/Makefile b/Makefile index bbd8ae81ff6..436b140b975 100644 --- a/Makefile +++ b/Makefile @@ -470,8 +470,8 @@ Q = @ endif CORE_VERSION = 10.0.0 -CPP_VERSION = 1.29.0-dev -CSHARP_VERSION = 2.29.0-dev +CPP_VERSION = 1.30.0-dev +CSHARP_VERSION = 2.30.0-dev CPPFLAGS_NO_ARCH += $(addprefix -I, $(INCLUDES)) $(addprefix -D, $(DEFINES)) CPPFLAGS += $(CPPFLAGS_NO_ARCH) $(ARCH_FLAGS) diff --git a/build_handwritten.yaml b/build_handwritten.yaml index 0b572a91ee0..eaf209d1db1 100644 --- a/build_handwritten.yaml +++ b/build_handwritten.yaml @@ -14,8 +14,8 @@ settings: '#10': See the expand_version.py for all the quirks here core_version: 10.0.0 csharp_major_version: 2 - g_stands_for: gringotts - version: 1.29.0-dev + g_stands_for: gradius + version: 1.30.0-dev targets: - name: check_epollexclusive build: tool diff --git a/doc/g_stands_for.md b/doc/g_stands_for.md index 382ccbf56f8..0ae28e66f57 100644 --- a/doc/g_stands_for.md +++ b/doc/g_stands_for.md @@ -28,4 +28,5 @@ - 1.26 'g' stands for ['gon'](https://github.com/grpc/grpc/tree/v1.26.x) - 1.27 'g' stands for ['guantao'](https://github.com/grpc/grpc/tree/v1.27.x) - 1.28 'g' stands for ['galactic'](https://github.com/grpc/grpc/tree/v1.28.x) -- 1.29 'g' stands for ['gringotts'](https://github.com/grpc/grpc/tree/master) +- 1.29 'g' stands for ['gringotts'](https://github.com/grpc/grpc/tree/v1.29.x) +- 1.30 'g' stands for ['gradius'](https://github.com/grpc/grpc/tree/master) diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index a3c35cbc912..95ba2391077 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -22,7 +22,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-C++' # TODO (mxyan): use version that match gRPC version when pod is stabilized - version = '1.29.0-dev' + version = '1.30.0-dev' s.version = version s.summary = 'gRPC C++ library' s.homepage = 'https://grpc.io' diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 8bf16911cc6..15b284a2675 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-Core' - version = '1.29.0-dev' + version = '1.30.0-dev' s.version = version s.summary = 'Core cross-platform gRPC library, written in C' s.homepage = 'https://grpc.io' diff --git a/gRPC-ProtoRPC.podspec b/gRPC-ProtoRPC.podspec index 14389f87089..ff1576d71e8 100644 --- a/gRPC-ProtoRPC.podspec +++ b/gRPC-ProtoRPC.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-ProtoRPC' - version = '1.29.0-dev' + version = '1.30.0-dev' s.version = version s.summary = 'RPC library for Protocol Buffers, based on gRPC' s.homepage = 'https://grpc.io' diff --git a/gRPC-RxLibrary.podspec b/gRPC-RxLibrary.podspec index 4f061d9bbc2..551909e8c89 100644 --- a/gRPC-RxLibrary.podspec +++ b/gRPC-RxLibrary.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-RxLibrary' - version = '1.29.0-dev' + version = '1.30.0-dev' s.version = version s.summary = 'Reactive Extensions library for iOS/OSX.' s.homepage = 'https://grpc.io' diff --git a/gRPC.podspec b/gRPC.podspec index 53a715e6072..194ddcd9f0c 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -20,7 +20,7 @@ Pod::Spec.new do |s| s.name = 'gRPC' - version = '1.29.0-dev' + version = '1.30.0-dev' s.version = version s.summary = 'gRPC client library for iOS/OSX' s.homepage = 'https://grpc.io' diff --git a/package.xml b/package.xml index 42f91a37b8c..6ce43b0d564 100644 --- a/package.xml +++ b/package.xml @@ -13,8 +13,8 @@ 2019-09-24 - 1.29.0dev - 1.29.0dev + 1.30.0dev + 1.30.0dev beta @@ -22,7 +22,7 @@ Apache 2.0 -- gRPC Core 1.29.0 update +- gRPC Core 1.30.0 update diff --git a/src/core/lib/surface/version.cc b/src/core/lib/surface/version.cc index b16e8ddedf2..207fbfe6ca5 100644 --- a/src/core/lib/surface/version.cc +++ b/src/core/lib/surface/version.cc @@ -25,4 +25,4 @@ const char* grpc_version_string(void) { return "10.0.0"; } -const char* grpc_g_stands_for(void) { return "gringotts"; } +const char* grpc_g_stands_for(void) { return "gradius"; } diff --git a/src/cpp/common/version_cc.cc b/src/cpp/common/version_cc.cc index 208fcd06113..cefbd9e6556 100644 --- a/src/cpp/common/version_cc.cc +++ b/src/cpp/common/version_cc.cc @@ -22,5 +22,5 @@ #include namespace grpc { -grpc::string Version() { return "1.29.0-dev"; } +grpc::string Version() { return "1.30.0-dev"; } } // namespace grpc diff --git a/src/csharp/Grpc.Core.Api/VersionInfo.cs b/src/csharp/Grpc.Core.Api/VersionInfo.cs index 7a61bc7f21e..bd4f3e1926e 100644 --- a/src/csharp/Grpc.Core.Api/VersionInfo.cs +++ b/src/csharp/Grpc.Core.Api/VersionInfo.cs @@ -33,11 +33,11 @@ namespace Grpc.Core /// /// Current AssemblyFileVersion of gRPC C# assemblies /// - public const string CurrentAssemblyFileVersion = "2.29.0.0"; + public const string CurrentAssemblyFileVersion = "2.30.0.0"; /// /// Current version of gRPC C# /// - public const string CurrentVersion = "2.29.0-dev"; + public const string CurrentVersion = "2.30.0-dev"; } } diff --git a/src/csharp/build/dependencies.props b/src/csharp/build/dependencies.props index 355cfcf7b40..9e85adf72f5 100644 --- a/src/csharp/build/dependencies.props +++ b/src/csharp/build/dependencies.props @@ -1,7 +1,7 @@ - 2.29.0-dev + 2.30.0-dev 3.11.4 diff --git a/src/csharp/build_unitypackage.bat b/src/csharp/build_unitypackage.bat index 4129a00bdb9..72e8fcc5cd9 100644 --- a/src/csharp/build_unitypackage.bat +++ b/src/csharp/build_unitypackage.bat @@ -13,7 +13,7 @@ @rem limitations under the License. @rem Current package versions -set VERSION=2.29.0-dev +set VERSION=2.30.0-dev @rem Adjust the location of nuget.exe set NUGET=C:\nuget\nuget.exe diff --git a/src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec index 1385b2b396c..ca1490bcc0a 100644 --- a/src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec @@ -42,7 +42,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler-gRPCCppPlugin' - v = '1.29.0-dev' + v = '1.30.0-dev' s.version = v s.summary = 'The gRPC ProtoC plugin generates C++ files from .proto services.' s.description = <<-DESC diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index 9ab5bba68e3..e48d3de8828 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -42,7 +42,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler-gRPCPlugin' - v = '1.29.0-dev' + v = '1.30.0-dev' s.version = v s.summary = 'The gRPC ProtoC plugin generates Objective-C files from .proto services.' s.description = <<-DESC diff --git a/src/objective-c/GRPCClient/version.h b/src/objective-c/GRPCClient/version.h index 43c7a0bcdeb..8b399a26ca9 100644 --- a/src/objective-c/GRPCClient/version.h +++ b/src/objective-c/GRPCClient/version.h @@ -22,4 +22,4 @@ // instead. This file can be regenerated from the template by running // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.29.0-dev" +#define GRPC_OBJC_VERSION_STRING @"1.30.0-dev" diff --git a/src/objective-c/tests/version.h b/src/objective-c/tests/version.h index 64d3191a7d4..78714dd7968 100644 --- a/src/objective-c/tests/version.h +++ b/src/objective-c/tests/version.h @@ -22,5 +22,5 @@ // instead. This file can be regenerated from the template by running // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.29.0-dev" +#define GRPC_OBJC_VERSION_STRING @"1.30.0-dev" #define GRPC_C_VERSION_STRING @"10.0.0" diff --git a/src/php/composer.json b/src/php/composer.json index 24ed00401b9..41a7d9ad093 100644 --- a/src/php/composer.json +++ b/src/php/composer.json @@ -2,7 +2,7 @@ "name": "grpc/grpc-dev", "description": "gRPC library for PHP - for Development use only", "license": "Apache-2.0", - "version": "1.29.0", + "version": "1.30.0", "require": { "php": ">=5.5.0", "google/protobuf": "^v3.3.0" diff --git a/src/php/ext/grpc/version.h b/src/php/ext/grpc/version.h index 817ec68c1f2..6e60f2cdb27 100644 --- a/src/php/ext/grpc/version.h +++ b/src/php/ext/grpc/version.h @@ -20,6 +20,6 @@ #ifndef VERSION_H #define VERSION_H -#define PHP_GRPC_VERSION "1.29.0dev" +#define PHP_GRPC_VERSION "1.30.0dev" #endif /* VERSION_H */ diff --git a/src/python/grpcio/grpc/_grpcio_metadata.py b/src/python/grpcio/grpc/_grpcio_metadata.py index b6c1f3b1ccd..a483c9fbd65 100644 --- a/src/python/grpcio/grpc/_grpcio_metadata.py +++ b/src/python/grpcio/grpc/_grpcio_metadata.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc/_grpcio_metadata.py.template`!!! -__version__ = """1.29.0.dev0""" +__version__ = """1.30.0.dev0""" diff --git a/src/python/grpcio/grpc_version.py b/src/python/grpcio/grpc_version.py index 4a2da07dce9..f03b1b74439 100644 --- a/src/python/grpcio/grpc_version.py +++ b/src/python/grpcio/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_version.py.template`!!! -VERSION = '1.29.0.dev0' +VERSION = '1.30.0.dev0' diff --git a/src/python/grpcio_channelz/grpc_version.py b/src/python/grpcio_channelz/grpc_version.py index 41f84117965..bb30eefa837 100644 --- a/src/python/grpcio_channelz/grpc_version.py +++ b/src/python/grpcio_channelz/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_channelz/grpc_version.py.template`!!! -VERSION = '1.29.0.dev0' +VERSION = '1.30.0.dev0' diff --git a/src/python/grpcio_health_checking/grpc_version.py b/src/python/grpcio_health_checking/grpc_version.py index 33b5ed79c86..b4c9f7652da 100644 --- a/src/python/grpcio_health_checking/grpc_version.py +++ b/src/python/grpcio_health_checking/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_health_checking/grpc_version.py.template`!!! -VERSION = '1.29.0.dev0' +VERSION = '1.30.0.dev0' diff --git a/src/python/grpcio_reflection/grpc_version.py b/src/python/grpcio_reflection/grpc_version.py index 09dc3617bbd..f370320520a 100644 --- a/src/python/grpcio_reflection/grpc_version.py +++ b/src/python/grpcio_reflection/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_reflection/grpc_version.py.template`!!! -VERSION = '1.29.0.dev0' +VERSION = '1.30.0.dev0' diff --git a/src/python/grpcio_status/grpc_version.py b/src/python/grpcio_status/grpc_version.py index 3d4010d2f9c..7956a946b7b 100644 --- a/src/python/grpcio_status/grpc_version.py +++ b/src/python/grpcio_status/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_status/grpc_version.py.template`!!! -VERSION = '1.29.0.dev0' +VERSION = '1.30.0.dev0' diff --git a/src/python/grpcio_testing/grpc_version.py b/src/python/grpcio_testing/grpc_version.py index ee6e1b52605..54753d30de8 100644 --- a/src/python/grpcio_testing/grpc_version.py +++ b/src/python/grpcio_testing/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_testing/grpc_version.py.template`!!! -VERSION = '1.29.0.dev0' +VERSION = '1.30.0.dev0' diff --git a/src/python/grpcio_tests/grpc_version.py b/src/python/grpcio_tests/grpc_version.py index efa361e32f3..d532daac8ce 100644 --- a/src/python/grpcio_tests/grpc_version.py +++ b/src/python/grpcio_tests/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_tests/grpc_version.py.template`!!! -VERSION = '1.29.0.dev0' +VERSION = '1.30.0.dev0' diff --git a/src/ruby/lib/grpc/version.rb b/src/ruby/lib/grpc/version.rb index f9e332f9eaa..40d0994cb33 100644 --- a/src/ruby/lib/grpc/version.rb +++ b/src/ruby/lib/grpc/version.rb @@ -14,5 +14,5 @@ # GRPC contains the General RPC module. module GRPC - VERSION = '1.29.0.dev' + VERSION = '1.30.0.dev' end diff --git a/src/ruby/tools/version.rb b/src/ruby/tools/version.rb index d43814ef8a6..561468ca85b 100644 --- a/src/ruby/tools/version.rb +++ b/src/ruby/tools/version.rb @@ -14,6 +14,6 @@ module GRPC module Tools - VERSION = '1.29.0.dev' + VERSION = '1.30.0.dev' end end diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index b561bb5ba1c..5cf5d94839c 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION = '1.29.0.dev0' +VERSION = '1.30.0.dev0' diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 412d506e53e..46959747465 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.29.0-dev +PROJECT_NUMBER = 1.30.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 94b242dbeb6..1761430d9e4 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.29.0-dev +PROJECT_NUMBER = 1.30.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.objc b/tools/doxygen/Doxyfile.objc index e23315f40cb..bb1645f8305 100644 --- a/tools/doxygen/Doxyfile.objc +++ b/tools/doxygen/Doxyfile.objc @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC Objective-C" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.29.0-dev +PROJECT_NUMBER = 1.30.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.objc.internal b/tools/doxygen/Doxyfile.objc.internal index 80dd661e4cd..a78d5b6d86c 100644 --- a/tools/doxygen/Doxyfile.objc.internal +++ b/tools/doxygen/Doxyfile.objc.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC Objective-C" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.29.0-dev +PROJECT_NUMBER = 1.30.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a From 1e9dc62db8ade88b0537eb857e0dd63d8a210b4f Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Thu, 30 Apr 2020 07:34:50 -0700 Subject: [PATCH 33/47] weighted_target: Create all children before updating any of them. --- .../weighted_target/weighted_target.cc | 20 ++++++++++++++----- test/cpp/end2end/xds_end2end_test.cc | 16 +++++++++++++++ 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc b/src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc index a0b0fef04a3..acbbcf0f957 100644 --- a/src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc +++ b/src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc @@ -278,19 +278,29 @@ void WeightedTargetLb::UpdateLocked(UpdateArgs args) { child->DeactivateLocked(); } } - // Add or update the targets in the new config. - HierarchicalAddressMap address_map = - MakeHierarchicalAddressMap(args.addresses); + // Create any children that don't already exist. + // Note that we add all children before updating any of them, because + // an update may trigger a child to immediately update its + // connectivity state (e.g., reporting TRANSIENT_FAILURE immediately when + // receiving an empty address list), and we don't want to return an + // overall state with incomplete data. for (const auto& p : config_->target_map()) { const std::string& name = p.first; - const WeightedTargetLbConfig::ChildConfig& config = p.second; auto it = targets_.find(name); if (it == targets_.end()) { it = targets_.emplace(std::make_pair(name, nullptr)).first; it->second = MakeOrphanable( Ref(DEBUG_LOCATION, "WeightedChild"), it->first); } - it->second->UpdateLocked(config, std::move(address_map[name]), args.args); + } + // Update all children. + HierarchicalAddressMap address_map = + MakeHierarchicalAddressMap(args.addresses); + for (const auto& p : config_->target_map()) { + const std::string& name = p.first; + const WeightedTargetLbConfig::ChildConfig& config = p.second; + targets_[name]->UpdateLocked(config, std::move(address_map[name]), + args.args); } } diff --git a/test/cpp/end2end/xds_end2end_test.cc b/test/cpp/end2end/xds_end2end_test.cc index f993bb88e78..0b4d671fa84 100644 --- a/test/cpp/end2end/xds_end2end_test.cc +++ b/test/cpp/end2end/xds_end2end_test.cc @@ -3196,6 +3196,22 @@ TEST_P(FailoverTest, DoesNotUsePriorityWithNoEndpoints) { } } +// Does not choose locality with no endpoints. +TEST_P(FailoverTest, DoesNotUseLocalityWithNoEndpoints) { + SetNextResolution({}); + SetNextResolutionForLbChannelAllBalancers(); + AdsServiceImpl::EdsResourceArgs args({ + {"locality0", {}, kDefaultLocalityWeight, 0}, + {"locality1", GetBackendPorts(), kDefaultLocalityWeight, 0}, + }); + balancers_[0]->ads_service()->SetEdsResource( + AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + // Wait for all backends to be used. + std::tuple counts = WaitForAllBackends(); + // Make sure no RPCs failed in the transition. + EXPECT_EQ(0, std::get<1>(counts)); +} + // If the higher priority localities are not reachable, failover to the highest // priority among the rest. TEST_P(FailoverTest, Failover) { From 0324afa89eaff68c6103ed725e50fdf468f8ced1 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Thu, 30 Apr 2020 10:09:03 -0700 Subject: [PATCH 34/47] grpclb: Add test showing that backend status is returned to client. --- test/cpp/end2end/grpclb_end2end_test.cc | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/test/cpp/end2end/grpclb_end2end_test.cc b/test/cpp/end2end/grpclb_end2end_test.cc index 3d5b6f8cc2c..2e3b4e9ca78 100644 --- a/test/cpp/end2end/grpclb_end2end_test.cc +++ b/test/cpp/end2end/grpclb_end2end_test.cc @@ -618,11 +618,17 @@ class GrpclbEnd2endTest : public ::testing::Test { } Status SendRpc(EchoResponse* response = nullptr, int timeout_ms = 1000, - bool wait_for_ready = false) { + bool wait_for_ready = false, + const Status& expected_status = Status::OK) { const bool local_response = (response == nullptr); if (local_response) response = new EchoResponse; EchoRequest request; request.set_message(kRequestMessage_); + if (!expected_status.ok()) { + auto* error = request.mutable_param()->mutable_expected_error(); + error->set_code(expected_status.error_code()); + error->set_error_message(expected_status.error_message()); + } ClientContext context; context.set_deadline(grpc_timeout_milliseconds_to_deadline(timeout_ms)); if (wait_for_ready) context.set_wait_for_ready(true); @@ -751,6 +757,22 @@ TEST_F(SingleBalancerTest, Vanilla) { EXPECT_EQ("grpclb", channel_->GetLoadBalancingPolicyName()); } +TEST_F(SingleBalancerTest, ReturnServerStatus) { + SetNextResolutionAllBalancers(); + ScheduleResponseForBalancer( + 0, BalancerServiceImpl::BuildResponseForBackends(GetBackendPorts(), {}), + 0); + // We need to wait for all backends to come online. + WaitForAllBackends(); + // Send a request that the backend will fail, and make sure we get + // back the right status. + Status expected(StatusCode::INVALID_ARGUMENT, "He's dead, Jim!"); + Status actual = SendRpc(/*response=*/nullptr, /*timeout_ms=*/1000, + /*wait_for_ready=*/false, expected); + EXPECT_EQ(actual.error_code(), expected.error_code()); + EXPECT_EQ(actual.error_message(), expected.error_message()); +} + TEST_F(SingleBalancerTest, SelectGrpclbWithMigrationServiceConfig) { SetNextResolutionAllBalancers( "{\n" From 9793af2b0923670bb3c2cb2bc47ba8a25fa31002 Mon Sep 17 00:00:00 2001 From: Donna Dionne Date: Thu, 30 Apr 2020 11:55:02 -0700 Subject: [PATCH 35/47] Consolidating LDS and RDS tests together And refactoring the code to be able to reuse the same helper methods. Updating all tests to use helper methods. --- test/cpp/end2end/xds_end2end_test.cc | 351 ++++++++------------------- 1 file changed, 105 insertions(+), 246 deletions(-) diff --git a/test/cpp/end2end/xds_end2end_test.cc b/test/cpp/end2end/xds_end2end_test.cc index 0b4d671fa84..c109c6834a3 100644 --- a/test/cpp/end2end/xds_end2end_test.cc +++ b/test/cpp/end2end/xds_end2end_test.cc @@ -1074,22 +1074,27 @@ class LrsServiceImpl : public LrsService, class TestType { public: - TestType(bool use_xds_resolver, bool enable_load_reporting) + TestType(bool use_xds_resolver, bool enable_load_reporting, + bool enable_rds_testing = false) : use_xds_resolver_(use_xds_resolver), - enable_load_reporting_(enable_load_reporting) {} + enable_load_reporting_(enable_load_reporting), + enable_rds_testing_(enable_rds_testing) {} bool use_xds_resolver() const { return use_xds_resolver_; } bool enable_load_reporting() const { return enable_load_reporting_; } + bool enable_rds_testing() const { return enable_rds_testing_; } grpc::string AsString() const { grpc::string retval = (use_xds_resolver_ ? "XdsResolver" : "FakeResolver"); if (enable_load_reporting_) retval += "WithLoadReporting"; + if (enable_rds_testing_) retval += "Rds"; return retval; } private: const bool use_xds_resolver_; const bool enable_load_reporting_; + const bool enable_rds_testing_; }; class XdsEnd2endTest : public ::testing::TestWithParam { @@ -1133,6 +1138,9 @@ class XdsEnd2endTest : public ::testing::TestWithParam { ? client_load_reporting_interval_seconds_ : 0)); balancers_.back()->Start(); + if (GetParam().enable_rds_testing()) { + balancers_[i]->ads_service()->SetLdsToUseDynamicRds(); + } } ResetStub(); } @@ -1468,6 +1476,24 @@ class XdsEnd2endTest : public ::testing::TestWithParam { } } + void SetRouteConfiguration(int idx, const RouteConfiguration& route_config) { + if (GetParam().enable_rds_testing()) { + balancers_[idx]->ads_service()->SetRdsResource(route_config, + kDefaultResourceName); + } else { + balancers_[idx]->ads_service()->SetLdsResource( + AdsServiceImpl::BuildListener(route_config), kDefaultResourceName); + } + } + + AdsServiceImpl::ResponseState RouteConfigurationResponseState(int idx) const { + AdsServiceImpl* ads_service = balancers_[idx]->ads_service(); + if (GetParam().enable_rds_testing()) { + return ads_service->rds_response_state(); + } + return ads_service->lds_response_state(); + } + public: // This method could benefit test subclasses; to make it accessible // via bind with a qualified name, it needs to be public. @@ -2128,16 +2154,6 @@ TEST_P(SecureNamingTest, TargetNameIsUnexpected) { using LdsTest = BasicTest; -// Tests that LDS client should send an ACK upon correct LDS response (with -// inlined RDS result). -TEST_P(LdsTest, Vanilla) { - SetNextResolution({}); - SetNextResolutionForLbChannelAllBalancers(); - (void)SendRpc(); - EXPECT_EQ(balancers_[0]->ads_service()->lds_response_state(), - AdsServiceImpl::ACKED); -} - // Tests that LDS client should send a NACK if there is no API listener in the // Listener in the LDS response. TEST_P(LdsTest, NoApiListener) { @@ -2167,25 +2183,34 @@ TEST_P(LdsTest, WrongRouteSpecifier) { AdsServiceImpl::NACKED); } +using LdsRdsTest = BasicTest; + +// Tests that LDS client should send an ACK upon correct LDS response (with +// inlined RDS result). +TEST_P(LdsRdsTest, Vanilla) { + SetNextResolution({}); + SetNextResolutionForLbChannelAllBalancers(); + (void)SendRpc(); + EXPECT_EQ(RouteConfigurationResponseState(0), AdsServiceImpl::ACKED); +} + // Tests that LDS client should send a NACK if matching domain can't be found in // the LDS response. -TEST_P(LdsTest, NoMatchedDomain) { +TEST_P(LdsRdsTest, NoMatchedDomain) { RouteConfiguration route_config = balancers_[0]->ads_service()->default_route_config(); route_config.mutable_virtual_hosts(0)->clear_domains(); route_config.mutable_virtual_hosts(0)->add_domains("unmatched_domain"); - balancers_[0]->ads_service()->SetLdsResource( - AdsServiceImpl::BuildListener(route_config), kDefaultResourceName); + SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); - EXPECT_EQ(balancers_[0]->ads_service()->lds_response_state(), - AdsServiceImpl::NACKED); + EXPECT_EQ(RouteConfigurationResponseState(0), AdsServiceImpl::NACKED); } // Tests that LDS client should choose the virtual host with matching domain if // multiple virtual hosts exist in the LDS response. -TEST_P(LdsTest, ChooseMatchedDomain) { +TEST_P(LdsRdsTest, ChooseMatchedDomain) { RouteConfiguration route_config = balancers_[0]->ads_service()->default_route_config(); *(route_config.add_virtual_hosts()) = route_config.virtual_hosts(0); @@ -2195,18 +2220,16 @@ TEST_P(LdsTest, ChooseMatchedDomain) { ->mutable_routes(0) ->mutable_route() ->mutable_cluster_header(); - balancers_[0]->ads_service()->SetLdsResource( - AdsServiceImpl::BuildListener(route_config), kDefaultResourceName); + SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); (void)SendRpc(); - EXPECT_EQ(balancers_[0]->ads_service()->lds_response_state(), - AdsServiceImpl::ACKED); + EXPECT_EQ(RouteConfigurationResponseState(0), AdsServiceImpl::ACKED); } // Tests that LDS client should choose the last route in the virtual host if // multiple routes exist in the LDS response. -TEST_P(LdsTest, ChooseLastRoute) { +TEST_P(LdsRdsTest, ChooseLastRoute) { RouteConfiguration route_config = balancers_[0]->ads_service()->default_route_config(); *(route_config.mutable_virtual_hosts(0)->add_routes()) = @@ -2215,36 +2238,32 @@ TEST_P(LdsTest, ChooseLastRoute) { ->mutable_routes(0) ->mutable_route() ->mutable_cluster_header(); - balancers_[0]->ads_service()->SetLdsResource( - AdsServiceImpl::BuildListener(route_config), kDefaultResourceName); + SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); (void)SendRpc(); - EXPECT_EQ(balancers_[0]->ads_service()->lds_response_state(), - AdsServiceImpl::ACKED); + EXPECT_EQ(RouteConfigurationResponseState(0), AdsServiceImpl::ACKED); } // Tests that LDS client should send a NACK if route match has non-empty prefix // as the only route (default) in the LDS response. -TEST_P(LdsTest, RouteMatchHasNonemptyPrefix) { +TEST_P(LdsRdsTest, RouteMatchHasNonemptyPrefix) { RouteConfiguration route_config = balancers_[0]->ads_service()->default_route_config(); route_config.mutable_virtual_hosts(0) ->mutable_routes(0) ->mutable_match() ->set_prefix("nonempty_prefix"); - balancers_[0]->ads_service()->SetLdsResource( - AdsServiceImpl::BuildListener(route_config), kDefaultResourceName); + SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); - EXPECT_EQ(balancers_[0]->ads_service()->lds_response_state(), - AdsServiceImpl::NACKED); + EXPECT_EQ(RouteConfigurationResponseState(0), AdsServiceImpl::NACKED); } // Tests that LDS client should send a NACK if route match has a prefix // string with no "/". -TEST_P(LdsTest, RouteMatchHasInvalidPrefixNonEmptyNoSlash) { +TEST_P(LdsRdsTest, RouteMatchHasInvalidPrefixNonEmptyNoSlash) { ResetStub(/*failover_timeout=*/0, /*expected_targets=*/"", /*xds_resource_does_not_exist_timeout*/ 0, @@ -2256,18 +2275,16 @@ TEST_P(LdsTest, RouteMatchHasInvalidPrefixNonEmptyNoSlash) { auto* default_route = route_config.mutable_virtual_hosts(0)->add_routes(); default_route->mutable_match()->set_prefix(""); default_route->mutable_route()->set_cluster(kDefaultResourceName); - balancers_[0]->ads_service()->SetLdsResource( - AdsServiceImpl::BuildListener(route_config), kDefaultResourceName); + SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); - EXPECT_EQ(balancers_[0]->ads_service()->lds_response_state(), - AdsServiceImpl::NACKED); + EXPECT_EQ(RouteConfigurationResponseState(0), AdsServiceImpl::NACKED); } // Tests that LDS client should send a NACK if route match has a prefix // string does not end with "/". -TEST_P(LdsTest, RouteMatchHasInvalidPrefixNoEndingSlash) { +TEST_P(LdsRdsTest, RouteMatchHasInvalidPrefixNoEndingSlash) { ResetStub(/*failover_timeout=*/0, /*expected_targets=*/"", /*xds_resource_does_not_exist_timeout*/ 0, @@ -2276,18 +2293,16 @@ TEST_P(LdsTest, RouteMatchHasInvalidPrefixNoEndingSlash) { balancers_[0]->ads_service()->default_route_config(); auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0); route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service"); - balancers_[0]->ads_service()->SetLdsResource( - AdsServiceImpl::BuildListener(route_config), kDefaultResourceName); + SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); - EXPECT_EQ(balancers_[0]->ads_service()->lds_response_state(), - AdsServiceImpl::NACKED); + EXPECT_EQ(RouteConfigurationResponseState(0), AdsServiceImpl::NACKED); } // Tests that LDS client should send a NACK if route match has a prefix // string does not start with "/". -TEST_P(LdsTest, RouteMatchHasInvalidPrefixNoLeadingSlash) { +TEST_P(LdsRdsTest, RouteMatchHasInvalidPrefixNoLeadingSlash) { ResetStub(/*failover_timeout=*/0, /*expected_targets=*/"", /*xds_resource_does_not_exist_timeout*/ 0, @@ -2296,18 +2311,16 @@ TEST_P(LdsTest, RouteMatchHasInvalidPrefixNoLeadingSlash) { balancers_[0]->ads_service()->default_route_config(); auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0); route1->mutable_match()->set_prefix("grpc.testing.EchoTest1Service/"); - balancers_[0]->ads_service()->SetLdsResource( - AdsServiceImpl::BuildListener(route_config), kDefaultResourceName); + SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); - EXPECT_EQ(balancers_[0]->ads_service()->lds_response_state(), - AdsServiceImpl::NACKED); + EXPECT_EQ(RouteConfigurationResponseState(0), AdsServiceImpl::NACKED); } // Tests that LDS client should send a NACK if route match has a prefix // string with extra content outside of "/service/". -TEST_P(LdsTest, RouteMatchHasInvalidPrefixExtraContent) { +TEST_P(LdsRdsTest, RouteMatchHasInvalidPrefixExtraContent) { ResetStub(/*failover_timeout=*/0, /*expected_targets=*/"", /*xds_resource_does_not_exist_timeout*/ 0, @@ -2316,18 +2329,16 @@ TEST_P(LdsTest, RouteMatchHasInvalidPrefixExtraContent) { balancers_[0]->ads_service()->default_route_config(); auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0); route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/Echo1"); - balancers_[0]->ads_service()->SetLdsResource( - AdsServiceImpl::BuildListener(route_config), kDefaultResourceName); + SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); - EXPECT_EQ(balancers_[0]->ads_service()->lds_response_state(), - AdsServiceImpl::NACKED); + EXPECT_EQ(RouteConfigurationResponseState(0), AdsServiceImpl::NACKED); } // Tests that LDS client should send a NACK if route match has a prefix // string "//". -TEST_P(LdsTest, RouteMatchHasInvalidPrefixNoContent) { +TEST_P(LdsRdsTest, RouteMatchHasInvalidPrefixNoContent) { ResetStub(/*failover_timeout=*/0, /*expected_targets=*/"", /*xds_resource_does_not_exist_timeout*/ 0, @@ -2336,18 +2347,16 @@ TEST_P(LdsTest, RouteMatchHasInvalidPrefixNoContent) { balancers_[0]->ads_service()->default_route_config(); auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0); route1->mutable_match()->set_prefix("//"); - balancers_[0]->ads_service()->SetLdsResource( - AdsServiceImpl::BuildListener(route_config), kDefaultResourceName); + SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); - EXPECT_EQ(balancers_[0]->ads_service()->lds_response_state(), - AdsServiceImpl::NACKED); + EXPECT_EQ(RouteConfigurationResponseState(0), AdsServiceImpl::NACKED); } // Tests that LDS client should send a NACK if route match has path // but it's empty. -TEST_P(LdsTest, RouteMatchHasInvalidPathEmptyPath) { +TEST_P(LdsRdsTest, RouteMatchHasInvalidPathEmptyPath) { ResetStub(/*failover_timeout=*/0, /*expected_targets=*/"", /*xds_resource_does_not_exist_timeout*/ 0, @@ -2359,18 +2368,16 @@ TEST_P(LdsTest, RouteMatchHasInvalidPathEmptyPath) { default_route->mutable_match()->set_prefix(""); default_route->mutable_route()->set_cluster(kDefaultResourceName); route1->mutable_match()->set_path(""); - balancers_[0]->ads_service()->SetLdsResource( - AdsServiceImpl::BuildListener(route_config), kDefaultResourceName); + SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); - EXPECT_EQ(balancers_[0]->ads_service()->lds_response_state(), - AdsServiceImpl::NACKED); + EXPECT_EQ(RouteConfigurationResponseState(0), AdsServiceImpl::NACKED); } // Tests that LDS client should send a NACK if route match has path // string does not start with "/". -TEST_P(LdsTest, RouteMatchHasInvalidPathNoLeadingSlash) { +TEST_P(LdsRdsTest, RouteMatchHasInvalidPathNoLeadingSlash) { ResetStub(/*failover_timeout=*/0, /*expected_targets=*/"", /*xds_resource_does_not_exist_timeout*/ 0, @@ -2382,18 +2389,16 @@ TEST_P(LdsTest, RouteMatchHasInvalidPathNoLeadingSlash) { default_route->mutable_match()->set_prefix(""); default_route->mutable_route()->set_cluster(kDefaultResourceName); route1->mutable_match()->set_path("grpc.testing.EchoTest1Service/Echo1"); - balancers_[0]->ads_service()->SetLdsResource( - AdsServiceImpl::BuildListener(route_config), kDefaultResourceName); + SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); - EXPECT_EQ(balancers_[0]->ads_service()->lds_response_state(), - AdsServiceImpl::NACKED); + EXPECT_EQ(RouteConfigurationResponseState(0), AdsServiceImpl::NACKED); } // Tests that LDS client should send a NACK if route match has path // string that ends with "/". -TEST_P(LdsTest, RouteMatchHasInvalidPathEndsWithSlash) { +TEST_P(LdsRdsTest, RouteMatchHasInvalidPathEndsWithSlash) { ResetStub(/*failover_timeout=*/0, /*expected_targets=*/"", /*xds_resource_does_not_exist_timeout*/ 0, @@ -2405,18 +2410,16 @@ TEST_P(LdsTest, RouteMatchHasInvalidPathEndsWithSlash) { default_route->mutable_match()->set_prefix(""); default_route->mutable_route()->set_cluster(kDefaultResourceName); route1->mutable_match()->set_path("/grpc.testing.EchoTest1Service/Echo1/"); - balancers_[0]->ads_service()->SetLdsResource( - AdsServiceImpl::BuildListener(route_config), kDefaultResourceName); + SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); - EXPECT_EQ(balancers_[0]->ads_service()->lds_response_state(), - AdsServiceImpl::NACKED); + EXPECT_EQ(RouteConfigurationResponseState(0), AdsServiceImpl::NACKED); } // Tests that LDS client should send a NACK if route match has path // string that misses "/" between service and method. -TEST_P(LdsTest, RouteMatchHasInvalidPathMissingMiddleSlash) { +TEST_P(LdsRdsTest, RouteMatchHasInvalidPathMissingMiddleSlash) { ResetStub(/*failover_timeout=*/0, /*expected_targets=*/"", /*xds_resource_does_not_exist_timeout*/ 0, @@ -2428,18 +2431,16 @@ TEST_P(LdsTest, RouteMatchHasInvalidPathMissingMiddleSlash) { default_route->mutable_match()->set_prefix(""); default_route->mutable_route()->set_cluster(kDefaultResourceName); route1->mutable_match()->set_path("/grpc.testing.EchoTest1Service.Echo1"); - balancers_[0]->ads_service()->SetLdsResource( - AdsServiceImpl::BuildListener(route_config), kDefaultResourceName); + SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); - EXPECT_EQ(balancers_[0]->ads_service()->lds_response_state(), - AdsServiceImpl::NACKED); + EXPECT_EQ(RouteConfigurationResponseState(0), AdsServiceImpl::NACKED); } // Tests that LDS client should send a NACK if route match has path // string that is missing service. -TEST_P(LdsTest, RouteMatchHasInvalidPathMissingService) { +TEST_P(LdsRdsTest, RouteMatchHasInvalidPathMissingService) { ResetStub(/*failover_timeout=*/0, /*expected_targets=*/"", /*xds_resource_does_not_exist_timeout*/ 0, @@ -2451,18 +2452,16 @@ TEST_P(LdsTest, RouteMatchHasInvalidPathMissingService) { default_route->mutable_match()->set_prefix(""); default_route->mutable_route()->set_cluster(kDefaultResourceName); route1->mutable_match()->set_path("//Echo1"); - balancers_[0]->ads_service()->SetLdsResource( - AdsServiceImpl::BuildListener(route_config), kDefaultResourceName); + SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); - EXPECT_EQ(balancers_[0]->ads_service()->lds_response_state(), - AdsServiceImpl::NACKED); + EXPECT_EQ(RouteConfigurationResponseState(0), AdsServiceImpl::NACKED); } // Tests that LDS client should send a NACK if route match has path // string that is missing method. -TEST_P(LdsTest, RouteMatchHasInvalidPathMissingMethod) { +TEST_P(LdsRdsTest, RouteMatchHasInvalidPathMissingMethod) { ResetStub(/*failover_timeout=*/0, /*expected_targets=*/"", /*xds_resource_does_not_exist_timeout*/ 0, @@ -2474,28 +2473,24 @@ TEST_P(LdsTest, RouteMatchHasInvalidPathMissingMethod) { default_route->mutable_match()->set_prefix(""); default_route->mutable_route()->set_cluster(kDefaultResourceName); route1->mutable_match()->set_path("/grpc.testing.EchoTest1Service/"); - balancers_[0]->ads_service()->SetLdsResource( - AdsServiceImpl::BuildListener(route_config), kDefaultResourceName); + SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); - EXPECT_EQ(balancers_[0]->ads_service()->lds_response_state(), - AdsServiceImpl::NACKED); + EXPECT_EQ(RouteConfigurationResponseState(0), AdsServiceImpl::NACKED); } // Tests that LDS client should send a NACK if route has an action other than // RouteAction in the LDS response. -TEST_P(LdsTest, RouteHasNoRouteAction) { +TEST_P(LdsRdsTest, RouteHasNoRouteAction) { RouteConfiguration route_config = balancers_[0]->ads_service()->default_route_config(); route_config.mutable_virtual_hosts(0)->mutable_routes(0)->mutable_redirect(); - balancers_[0]->ads_service()->SetLdsResource( - AdsServiceImpl::BuildListener(route_config), kDefaultResourceName); + SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); - EXPECT_EQ(balancers_[0]->ads_service()->lds_response_state(), - AdsServiceImpl::NACKED); + EXPECT_EQ(RouteConfigurationResponseState(0), AdsServiceImpl::NACKED); } // TODO@donnadionne: Add more invalid config tests to cover all errors in @@ -2503,26 +2498,28 @@ TEST_P(LdsTest, RouteHasNoRouteAction) { // Tests that LDS client should send a NACK if RouteAction has a // cluster_specifier other than cluster in the LDS response. -TEST_P(LdsTest, RouteActionHasNoCluster) { +TEST_P(LdsRdsTest, RouteActionHasNoCluster) { RouteConfiguration route_config = balancers_[0]->ads_service()->default_route_config(); route_config.mutable_virtual_hosts(0) ->mutable_routes(0) ->mutable_route() ->mutable_cluster_header(); - balancers_[0]->ads_service()->SetLdsResource( - AdsServiceImpl::BuildListener(route_config), kDefaultResourceName); + SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); - EXPECT_EQ(balancers_[0]->ads_service()->lds_response_state(), - AdsServiceImpl::NACKED); + EXPECT_EQ(RouteConfigurationResponseState(0), AdsServiceImpl::NACKED); } // Tests that LDS client times out when no response received. -TEST_P(LdsTest, Timeout) { +TEST_P(LdsRdsTest, Timeout) { ResetStub(0, "", 500); - balancers_[0]->ads_service()->SetResourceIgnore(kLdsTypeUrl); + if (GetParam().enable_rds_testing()) { + balancers_[0]->ads_service()->SetResourceIgnore(kRdsTypeUrl); + } else { + balancers_[0]->ads_service()->SetResourceIgnore(kLdsTypeUrl); + } SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); @@ -2530,7 +2527,7 @@ TEST_P(LdsTest, Timeout) { // Tests that LDS client should choose the default route (with no matching // specified) after unable to find a match with previous routes. -TEST_P(LdsTest, XdsRoutingPathMatching) { +TEST_P(LdsRdsTest, XdsRoutingPathMatching) { ResetStub(/*failover_timeout=*/0, /*expected_targets=*/"", /*xds_resource_does_not_exist_timeout*/ 0, @@ -2582,9 +2579,7 @@ TEST_P(LdsTest, XdsRoutingPathMatching) { auto* default_route = new_route_config.mutable_virtual_hosts(0)->add_routes(); default_route->mutable_match()->set_prefix(""); default_route->mutable_route()->set_cluster(kDefaultResourceName); - Listener listener = - balancers_[0]->ads_service()->BuildListener(new_route_config); - balancers_[0]->ads_service()->SetLdsResource(listener, kDefaultResourceName); + SetRouteConfiguration(0, new_route_config); WaitForAllBackends(0, 2); CheckRpcSendOk(kNumEchoRpcs, RpcOptions().set_wait_for_ready(true)); CheckRpcSendOk(kNumEcho1Rpcs, RpcOptions() @@ -2610,7 +2605,7 @@ TEST_P(LdsTest, XdsRoutingPathMatching) { EXPECT_EQ(kNumEcho2Rpcs, backends_[3]->backend_service2()->request_count()); } -TEST_P(LdsTest, XdsRoutingPrefixMatching) { +TEST_P(LdsRdsTest, XdsRoutingPrefixMatching) { ResetStub(/*failover_timeout=*/0, /*expected_targets=*/"", /*xds_resource_does_not_exist_timeout*/ 0, @@ -2659,9 +2654,7 @@ TEST_P(LdsTest, XdsRoutingPrefixMatching) { auto* default_route = new_route_config.mutable_virtual_hosts(0)->add_routes(); default_route->mutable_match()->set_prefix(""); default_route->mutable_route()->set_cluster(kDefaultResourceName); - Listener listener = - balancers_[0]->ads_service()->BuildListener(new_route_config); - balancers_[0]->ads_service()->SetLdsResource(listener, kDefaultResourceName); + SetRouteConfiguration(0, new_route_config); WaitForAllBackends(0, 2); CheckRpcSendOk(kNumEchoRpcs, RpcOptions().set_wait_for_ready(true)); CheckRpcSendOk( @@ -2685,142 +2678,6 @@ TEST_P(LdsTest, XdsRoutingPrefixMatching) { EXPECT_EQ(kNumEcho2Rpcs, backends_[3]->backend_service2()->request_count()); } -using RdsTest = BasicTest; - -// Tests that RDS client should send an ACK upon correct RDS response. -TEST_P(RdsTest, Vanilla) { - balancers_[0]->ads_service()->SetLdsToUseDynamicRds(); - SetNextResolution({}); - SetNextResolutionForLbChannelAllBalancers(); - (void)SendRpc(); - EXPECT_EQ(balancers_[0]->ads_service()->rds_response_state(), - AdsServiceImpl::ACKED); -} - -// Tests that RDS client should send a NACK if matching domain can't be found in -// the RDS response. -TEST_P(RdsTest, NoMatchedDomain) { - balancers_[0]->ads_service()->SetLdsToUseDynamicRds(); - RouteConfiguration route_config = - balancers_[0]->ads_service()->default_route_config(); - route_config.mutable_virtual_hosts(0)->clear_domains(); - route_config.mutable_virtual_hosts(0)->add_domains("unmatched_domain"); - balancers_[0]->ads_service()->SetRdsResource(route_config, - kDefaultResourceName); - SetNextResolution({}); - SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); - EXPECT_EQ(balancers_[0]->ads_service()->rds_response_state(), - AdsServiceImpl::NACKED); -} - -// Tests that RDS client should choose the virtual host with matching domain if -// multiple virtual hosts exist in the RDS response. -TEST_P(RdsTest, ChooseMatchedDomain) { - balancers_[0]->ads_service()->SetLdsToUseDynamicRds(); - RouteConfiguration route_config = - balancers_[0]->ads_service()->default_route_config(); - *(route_config.add_virtual_hosts()) = route_config.virtual_hosts(0); - route_config.mutable_virtual_hosts(0)->clear_domains(); - route_config.mutable_virtual_hosts(0)->add_domains("unmatched_domain"); - route_config.mutable_virtual_hosts(0) - ->mutable_routes(0) - ->mutable_route() - ->mutable_cluster_header(); - balancers_[0]->ads_service()->SetRdsResource(route_config, - kDefaultResourceName); - SetNextResolution({}); - SetNextResolutionForLbChannelAllBalancers(); - (void)SendRpc(); - EXPECT_EQ(balancers_[0]->ads_service()->rds_response_state(), - AdsServiceImpl::ACKED); -} - -// Tests that RDS client should choose the last route in the virtual host if -// multiple routes exist in the RDS response. -TEST_P(RdsTest, ChooseLastRoute) { - balancers_[0]->ads_service()->SetLdsToUseDynamicRds(); - RouteConfiguration route_config = - balancers_[0]->ads_service()->default_route_config(); - *(route_config.mutable_virtual_hosts(0)->add_routes()) = - route_config.virtual_hosts(0).routes(0); - route_config.mutable_virtual_hosts(0) - ->mutable_routes(0) - ->mutable_route() - ->mutable_cluster_header(); - balancers_[0]->ads_service()->SetRdsResource(route_config, - kDefaultResourceName); - SetNextResolution({}); - SetNextResolutionForLbChannelAllBalancers(); - (void)SendRpc(); - EXPECT_EQ(balancers_[0]->ads_service()->rds_response_state(), - AdsServiceImpl::ACKED); -} - -// Tests that RDS client should send a NACK if route match has non-empty prefix -// as the only route (default) in the RDS response. -TEST_P(RdsTest, RouteMatchHasNonemptyPrefix) { - balancers_[0]->ads_service()->SetLdsToUseDynamicRds(); - RouteConfiguration route_config = - balancers_[0]->ads_service()->default_route_config(); - route_config.mutable_virtual_hosts(0) - ->mutable_routes(0) - ->mutable_match() - ->set_prefix("/nonempty_prefix/"); - balancers_[0]->ads_service()->SetRdsResource(route_config, - kDefaultResourceName); - SetNextResolution({}); - SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); - EXPECT_EQ(balancers_[0]->ads_service()->rds_response_state(), - AdsServiceImpl::NACKED); -} - -// Tests that RDS client should send a NACK if route has an action other than -// RouteAction in the RDS response. -TEST_P(RdsTest, RouteHasNoRouteAction) { - balancers_[0]->ads_service()->SetLdsToUseDynamicRds(); - RouteConfiguration route_config = - balancers_[0]->ads_service()->default_route_config(); - route_config.mutable_virtual_hosts(0)->mutable_routes(0)->mutable_redirect(); - balancers_[0]->ads_service()->SetRdsResource(route_config, - kDefaultResourceName); - SetNextResolution({}); - SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); - EXPECT_EQ(balancers_[0]->ads_service()->rds_response_state(), - AdsServiceImpl::NACKED); -} - -// Tests that RDS client should send a NACK if RouteAction has a -// cluster_specifier other than cluster in the RDS response. -TEST_P(RdsTest, RouteActionHasNoCluster) { - balancers_[0]->ads_service()->SetLdsToUseDynamicRds(); - RouteConfiguration route_config = - balancers_[0]->ads_service()->default_route_config(); - route_config.mutable_virtual_hosts(0) - ->mutable_routes(0) - ->mutable_route() - ->mutable_cluster_header(); - balancers_[0]->ads_service()->SetRdsResource(route_config, - kDefaultResourceName); - SetNextResolution({}); - SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); - EXPECT_EQ(balancers_[0]->ads_service()->rds_response_state(), - AdsServiceImpl::NACKED); -} - -// Tests that RDS client times out when no response received. -TEST_P(RdsTest, Timeout) { - ResetStub(0, "", 500); - balancers_[0]->ads_service()->SetResourceIgnore(kRdsTypeUrl); - balancers_[0]->ads_service()->SetLdsToUseDynamicRds(); - SetNextResolution({}); - SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); -} - using CdsTest = BasicTest; // Tests that CDS client should send an ACK upon correct CDS response. @@ -4060,10 +3917,12 @@ INSTANTIATE_TEST_SUITE_P(XdsTest, LdsTest, TestType(true, true)), &TestTypeName); -// RDS depends on XdsResolver. -INSTANTIATE_TEST_SUITE_P(XdsTest, RdsTest, +// LDS RDS Commmon tests depends on XdsResolver. +INSTANTIATE_TEST_SUITE_P(XdsTest, LdsRdsTest, ::testing::Values(TestType(true, false), - TestType(true, true)), + TestType(true, true), + TestType(true, false, true), + TestType(true, true, true)), &TestTypeName); // CDS depends on XdsResolver. From f375c0b548278f81a21d3263cd00e57295cbf39a Mon Sep 17 00:00:00 2001 From: Taras Galkovskyi Date: Thu, 30 Apr 2020 18:58:44 +0000 Subject: [PATCH 36/47] minor stylistic fixes --- examples/python/data_transmission/BUILD | 2 +- examples/python/data_transmission/alts_server.py | 2 +- src/python/grpcio/grpc/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/python/data_transmission/BUILD b/examples/python/data_transmission/BUILD index 7c2980d5a5b..2eefb7a3ed2 100644 --- a/examples/python/data_transmission/BUILD +++ b/examples/python/data_transmission/BUILD @@ -1,4 +1,4 @@ -# Copyright 2017 gRPC authors. +# Copyright 2020 gRPC authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/examples/python/data_transmission/alts_server.py b/examples/python/data_transmission/alts_server.py index 9189405c916..7f1d2f8983d 100644 --- a/examples/python/data_transmission/alts_server.py +++ b/examples/python/data_transmission/alts_server.py @@ -1,4 +1,4 @@ -# Copyright 2019 gRPC authors. +# Copyright 2020 gRPC authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/python/grpcio/grpc/__init__.py b/src/python/grpcio/grpc/__init__.py index 258fb5cf52b..f4c261c23f2 100644 --- a/src/python/grpcio/grpc/__init__.py +++ b/src/python/grpcio/grpc/__init__.py @@ -1861,7 +1861,7 @@ def alts_server_credentials(): ALTS credentials API can only be used in GCP environment as it relies on handshaker service being available. For more info about ALTS see https://cloud.google.com/security/encryption-in-transit/application-layer-transport-security - + Returns: A ServerCredentials for use with an ALTS-enabled Server """ From 39d8fa3f0bc5491edc67dfaa49e8ea843360eab2 Mon Sep 17 00:00:00 2001 From: Pau Freixes Date: Thu, 30 Apr 2020 22:32:12 +0200 Subject: [PATCH 37/47] Apply feedback review --- .../grpcio/grpc/experimental/aio/_call.py | 2 +- .../grpcio/grpc/experimental/aio/_channel.py | 11 ++++++----- .../grpc/experimental/aio/_interceptor.py | 19 ++++++++++++------- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/python/grpcio/grpc/experimental/aio/_call.py b/src/python/grpcio/grpc/experimental/aio/_call.py index 9cab7052318..a0693921461 100644 --- a/src/python/grpcio/grpc/experimental/aio/_call.py +++ b/src/python/grpcio/grpc/experimental/aio/_call.py @@ -35,7 +35,7 @@ _LOCAL_CANCELLATION_DETAILS = 'Locally cancelled by application!' _GC_CANCELLATION_DETAILS = 'Cancelled upon garbage collection!' _RPC_ALREADY_FINISHED_DETAILS = 'RPC already finished.' _RPC_HALF_CLOSED_DETAILS = 'RPC is half closed after calling "done_writing".' -_API_STYLE_ERROR = 'Please don\'t mix two styles of API for streaming requests' +_API_STYLE_ERROR = 'The iterator and read/write APIs may not be mixed on a single RPC.' _OK_CALL_REPRESENTATION = ('<{} of RPC that terminated with:\n' '\tstatus = {}\n' diff --git a/src/python/grpcio/grpc/experimental/aio/_channel.py b/src/python/grpcio/grpc/experimental/aio/_channel.py index 731a3351b48..97ffa833bbf 100644 --- a/src/python/grpcio/grpc/experimental/aio/_channel.py +++ b/src/python/grpcio/grpc/experimental/aio/_channel.py @@ -256,11 +256,12 @@ class Channel(_base_channel.Channel): if invalid_interceptors: raise ValueError( - "Interceptor must be "+\ - "UnaryUnaryClientInterceptors or "+\ - "UnaryUnaryClientInterceptors or "+\ - "StreamUnaryClientInterceptors. The following are invalid: {}"\ - .format(invalid_interceptors)) + "Interceptor must be " + + "{} or ".format(UnaryUnaryClientInterceptor.__name__) + + "{} or ".format(UnaryStreamClientInterceptor.__name__) + + "{}. ".format(StreamUnaryClientInterceptor.__name__) + + "The following are invalid: {}".format(invalid_interceptors) + ) self._loop = asyncio.get_event_loop() self._channel = cygrpc.AioChannel( diff --git a/src/python/grpcio/grpc/experimental/aio/_interceptor.py b/src/python/grpcio/grpc/experimental/aio/_interceptor.py index eff5e03fc1e..e4969ddb4a5 100644 --- a/src/python/grpcio/grpc/experimental/aio/_interceptor.py +++ b/src/python/grpcio/grpc/experimental/aio/_interceptor.py @@ -104,7 +104,7 @@ class UnaryUnaryClientInterceptor(ClientInterceptor, metaclass=ABCMeta): Args: continuation: A coroutine that proceeds with the invocation by - executing the next interceptor in chain or invoking the + executing the next interceptor in the chain or invoking the actual RPC on the underlying Channel. It is the interceptor's responsibility to call it if it decides to move the RPC forward. The interceptor can use @@ -141,7 +141,7 @@ class UnaryStreamClientInterceptor(ClientInterceptor, metaclass=ABCMeta): Args: continuation: A coroutine that proceeds with the invocation by - executing the next interceptor in chain or invoking the + executing the next interceptor in the chain or invoking the actual RPC on the underlying Channel. It is the interceptor's responsibility to call it if it decides to move the RPC forward. The interceptor can use @@ -174,9 +174,14 @@ class StreamUnaryClientInterceptor(ClientInterceptor, metaclass=ABCMeta): ) -> StreamUnaryCall: """Intercepts a stream-unary invocation asynchronously. + Within the interceptor the usage of the call methods like `write` or + even awaiting the call should be done carefully, since the caller + could be expecting an untouched call, for example for start writing + messages to it. + Args: continuation: A coroutine that proceeds with the invocation by - executing the next interceptor in chain or invoking the + executing the next interceptor in the chain or invoking the actual RPC on the underlying Channel. It is the interceptor's responsibility to call it if it decides to move the RPC forward. The interceptor can use @@ -564,7 +569,7 @@ class InterceptedStreamUnaryCall(_InterceptedUnaryResponseMixin, _write_to_iterator_async_gen: Optional[AsyncIterable[RequestType]] _write_to_iterator_queue: Optional[asyncio.Queue] - _FINISH_ITERATOR_SENTINEL = tuple() + _FINISH_ITERATOR_SENTINEL = object() # pylint: disable=too-many-arguments def __init__(self, interceptors: Sequence[StreamUnaryClientInterceptor], @@ -577,11 +582,11 @@ class InterceptedStreamUnaryCall(_InterceptedUnaryResponseMixin, loop: asyncio.AbstractEventLoop) -> None: self._loop = loop self._channel = channel - if not request_iterator: + if request_iterator is None: # We provide our own request iterator which is a proxy # of the futures writes that will be done by the caller. self._write_to_iterator_queue = asyncio.Queue(maxsize=1) - self._write_to_iterator_async_gen = self._proxies_writes_as_a_request_iteerator( + self._write_to_iterator_async_gen = self._proxy_writes_as_request_iterator( ) request_iterator = self._write_to_iterator_async_gen else: @@ -636,7 +641,7 @@ class InterceptedStreamUnaryCall(_InterceptedUnaryResponseMixin, def time_remaining(self) -> Optional[float]: raise NotImplementedError() - async def _proxies_writes_as_a_request_iteerator(self): + async def _proxy_writes_as_request_iterator(self): await self._interceptors_task while True: From 092932b4dddbcdf20c8ec5aa9dc7c356b309b93f Mon Sep 17 00:00:00 2001 From: Donna Dionne Date: Thu, 30 Apr 2020 14:57:29 -0700 Subject: [PATCH 38/47] Squash the last 2 commits. --- .../ext/filters/client_channel/xds/xds_api.cc | 7 +- test/cpp/end2end/xds_end2end_test.cc | 231 +++++++++++------- 2 files changed, 147 insertions(+), 91 deletions(-) diff --git a/src/core/ext/filters/client_channel/xds/xds_api.cc b/src/core/ext/filters/client_channel/xds/xds_api.cc index 3c977d6152f..60ca930e060 100644 --- a/src/core/ext/filters/client_channel/xds/xds_api.cc +++ b/src/core/ext/filters/client_channel/xds/xds_api.cc @@ -1145,7 +1145,7 @@ grpc_error* LdsResponseParse(XdsClient* client, TraceFlag* tracer, envoy_api_v2_Listener_api_listener(listener); if (api_listener == nullptr) { return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Listener doesn't have ApiListener."); + "Listener has no ApiListener."); } const upb_strview encoded_api_listener = google_protobuf_Any_value( envoy_config_listener_v2_ApiListener_api_listener(api_listener)); @@ -1278,7 +1278,8 @@ grpc_error* CdsResponseParse( const envoy_api_v2_core_ConfigSource* eds_config = envoy_api_v2_Cluster_EdsClusterConfig_eds_config(eds_cluster_config); if (!envoy_api_v2_core_ConfigSource_has_ads(eds_config)) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING("ConfigSource is not ADS."); + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "EDS ConfigSource is not ADS."); } // Record EDS service_name (if any). upb_strview service_name = @@ -1299,7 +1300,7 @@ grpc_error* CdsResponseParse( if (lrs_server != nullptr) { if (!envoy_api_v2_core_ConfigSource_has_self(lrs_server)) { return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "ConfigSource is not self."); + "LRS ConfigSource is not self."); } cds_update.lrs_load_reporting_server_name.emplace(""); } diff --git a/test/cpp/end2end/xds_end2end_test.cc b/test/cpp/end2end/xds_end2end_test.cc index c109c6834a3..5fb6a62b19d 100644 --- a/test/cpp/end2end/xds_end2end_test.cc +++ b/test/cpp/end2end/xds_end2end_test.cc @@ -375,11 +375,10 @@ class ClientStats { class AdsServiceImpl : public AggregatedDiscoveryService::Service, public std::enable_shared_from_this { public: - enum ResponseState { - NOT_SENT, - SENT, - ACKED, - NACKED, + struct ResponseState { + enum State { NOT_SENT, SENT, ACKED, NACKED }; + State state = NOT_SENT; + std::string error_message; }; struct EdsResourceArgs { @@ -487,11 +486,15 @@ class AdsServiceImpl : public AggregatedDiscoveryService::Service, // comparing it to nonce (this server ensures they are always set to // the same in a response.) if (!request.response_nonce().empty()) { - resource_type_response_state_[request.type_url()] = + resource_type_response_state_[request.type_url()].state = (!request.version_info().empty() && request.version_info() == request.response_nonce()) - ? ACKED - : NACKED; + ? ResponseState::ACKED + : ResponseState::NACKED; + } + if (request.has_error_detail()) { + resource_type_response_state_[request.type_url()].error_message = + request.error_detail().message(); } // As long as the test did not tell us to ignore this type of // request, we will loop through all resources to: @@ -920,7 +923,7 @@ class AdsServiceImpl : public AggregatedDiscoveryService::Service, const SubscriptionNameMap& subscription_name_map, const std::set& resources_added_to_response, DiscoveryResponse* response) { - resource_type_response_state_[resource_type] = SENT; + resource_type_response_state_[resource_type].state = ResponseState::SENT; response->set_type_url(resource_type); response->set_version_info(absl::StrCat(version)); response->set_nonce(absl::StrCat(version)); @@ -949,7 +952,7 @@ class AdsServiceImpl : public AggregatedDiscoveryService::Service, Listener default_listener_; RouteConfiguration default_route_config_; Cluster default_cluster_; - std::map + std::map resource_type_response_state_; std::set resource_types_to_ignore_; // An instance data member containing the current state of all resources. @@ -1904,8 +1907,8 @@ TEST_P(XdsResolverOnlyTest, ClusterRemoved) { // Make sure RPCs are still succeeding. CheckRpcSendOk(100 * num_backends_); // Make sure we ACK'ed the update. - EXPECT_EQ(balancers_[0]->ads_service()->cds_response_state(), - AdsServiceImpl::ACKED); + EXPECT_EQ(balancers_[0]->ads_service()->cds_response_state().state, + AdsServiceImpl::ResponseState::ACKED); } // Tests that we restart all xDS requests when we reestablish the ADS call. @@ -2163,8 +2166,10 @@ TEST_P(LdsTest, NoApiListener) { SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); - EXPECT_EQ(balancers_[0]->ads_service()->lds_response_state(), - AdsServiceImpl::NACKED); + const auto& response_state = + balancers_[0]->ads_service()->lds_response_state(); + EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); + EXPECT_EQ(response_state.error_message, "Listener has no ApiListener."); } // Tests that LDS client should send a NACK if the route_specifier in the @@ -2179,8 +2184,11 @@ TEST_P(LdsTest, WrongRouteSpecifier) { SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); - EXPECT_EQ(balancers_[0]->ads_service()->lds_response_state(), - AdsServiceImpl::NACKED); + const auto& response_state = + balancers_[0]->ads_service()->lds_response_state(); + EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); + EXPECT_EQ(response_state.error_message, + "HttpConnectionManager neither has inlined route_config nor RDS."); } using LdsRdsTest = BasicTest; @@ -2191,7 +2199,8 @@ TEST_P(LdsRdsTest, Vanilla) { SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); (void)SendRpc(); - EXPECT_EQ(RouteConfigurationResponseState(0), AdsServiceImpl::ACKED); + EXPECT_EQ(RouteConfigurationResponseState(0).state, + AdsServiceImpl::ResponseState::ACKED); } // Tests that LDS client should send a NACK if matching domain can't be found in @@ -2205,7 +2214,10 @@ TEST_P(LdsRdsTest, NoMatchedDomain) { SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); - EXPECT_EQ(RouteConfigurationResponseState(0), AdsServiceImpl::NACKED); + const auto& response_state = RouteConfigurationResponseState(0); + EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); + EXPECT_EQ(response_state.error_message, + "No matched virtual host found in the route config."); } // Tests that LDS client should choose the virtual host with matching domain if @@ -2224,7 +2236,8 @@ TEST_P(LdsRdsTest, ChooseMatchedDomain) { SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); (void)SendRpc(); - EXPECT_EQ(RouteConfigurationResponseState(0), AdsServiceImpl::ACKED); + EXPECT_EQ(RouteConfigurationResponseState(0).state, + AdsServiceImpl::ResponseState::ACKED); } // Tests that LDS client should choose the last route in the virtual host if @@ -2242,7 +2255,8 @@ TEST_P(LdsRdsTest, ChooseLastRoute) { SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); (void)SendRpc(); - EXPECT_EQ(RouteConfigurationResponseState(0), AdsServiceImpl::ACKED); + EXPECT_EQ(RouteConfigurationResponseState(0).state, + AdsServiceImpl::ResponseState::ACKED); } // Tests that LDS client should send a NACK if route match has non-empty prefix @@ -2253,12 +2267,16 @@ TEST_P(LdsRdsTest, RouteMatchHasNonemptyPrefix) { route_config.mutable_virtual_hosts(0) ->mutable_routes(0) ->mutable_match() - ->set_prefix("nonempty_prefix"); + ->set_prefix("/nonempty_prefix/"); SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); - EXPECT_EQ(RouteConfigurationResponseState(0), AdsServiceImpl::NACKED); + const auto& response_state = RouteConfigurationResponseState(0); + balancers_[0]->ads_service()->lds_response_state(); + EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); + EXPECT_EQ(response_state.error_message, + "Default route must have empty service and method"); } // Tests that LDS client should send a NACK if route match has a prefix @@ -2279,7 +2297,9 @@ TEST_P(LdsRdsTest, RouteMatchHasInvalidPrefixNonEmptyNoSlash) { SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); - EXPECT_EQ(RouteConfigurationResponseState(0), AdsServiceImpl::NACKED); + const auto& response_state = RouteConfigurationResponseState(0); + EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); + EXPECT_EQ(response_state.error_message, "Prefix does not start with a /"); } // Tests that LDS client should send a NACK if route match has a prefix @@ -2297,7 +2317,10 @@ TEST_P(LdsRdsTest, RouteMatchHasInvalidPrefixNoEndingSlash) { SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); - EXPECT_EQ(RouteConfigurationResponseState(0), AdsServiceImpl::NACKED); + const auto& response_state = RouteConfigurationResponseState(0); + EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); + EXPECT_EQ(response_state.error_message, + "Prefix not in the required format of /service/"); } // Tests that LDS client should send a NACK if route match has a prefix @@ -2315,7 +2338,9 @@ TEST_P(LdsRdsTest, RouteMatchHasInvalidPrefixNoLeadingSlash) { SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); - EXPECT_EQ(RouteConfigurationResponseState(0), AdsServiceImpl::NACKED); + const auto& response_state = RouteConfigurationResponseState(0); + EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); + EXPECT_EQ(response_state.error_message, "Prefix does not start with a /"); } // Tests that LDS client should send a NACK if route match has a prefix @@ -2333,7 +2358,9 @@ TEST_P(LdsRdsTest, RouteMatchHasInvalidPrefixExtraContent) { SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); - EXPECT_EQ(RouteConfigurationResponseState(0), AdsServiceImpl::NACKED); + const auto& response_state = RouteConfigurationResponseState(0); + EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); + EXPECT_EQ(response_state.error_message, "Prefix does not end with a /"); } // Tests that LDS client should send a NACK if route match has a prefix @@ -2351,7 +2378,9 @@ TEST_P(LdsRdsTest, RouteMatchHasInvalidPrefixNoContent) { SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); - EXPECT_EQ(RouteConfigurationResponseState(0), AdsServiceImpl::NACKED); + const auto& response_state = RouteConfigurationResponseState(0); + EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); + EXPECT_EQ(response_state.error_message, "Prefix contains empty service name"); } // Tests that LDS client should send a NACK if route match has path @@ -2372,7 +2401,9 @@ TEST_P(LdsRdsTest, RouteMatchHasInvalidPathEmptyPath) { SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); - EXPECT_EQ(RouteConfigurationResponseState(0), AdsServiceImpl::NACKED); + const auto& response_state = RouteConfigurationResponseState(0); + EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); + EXPECT_EQ(response_state.error_message, "Path if set cannot be empty"); } // Tests that LDS client should send a NACK if route match has path @@ -2393,7 +2424,9 @@ TEST_P(LdsRdsTest, RouteMatchHasInvalidPathNoLeadingSlash) { SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); - EXPECT_EQ(RouteConfigurationResponseState(0), AdsServiceImpl::NACKED); + const auto& response_state = RouteConfigurationResponseState(0); + EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); + EXPECT_EQ(response_state.error_message, "Path does not start with a /"); } // Tests that LDS client should send a NACK if route match has path @@ -2414,7 +2447,10 @@ TEST_P(LdsRdsTest, RouteMatchHasInvalidPathEndsWithSlash) { SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); - EXPECT_EQ(RouteConfigurationResponseState(0), AdsServiceImpl::NACKED); + const auto& response_state = RouteConfigurationResponseState(0); + EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); + EXPECT_EQ(response_state.error_message, + "Path not in the required format of /service/method"); } // Tests that LDS client should send a NACK if route match has path @@ -2435,7 +2471,10 @@ TEST_P(LdsRdsTest, RouteMatchHasInvalidPathMissingMiddleSlash) { SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); - EXPECT_EQ(RouteConfigurationResponseState(0), AdsServiceImpl::NACKED); + const auto& response_state = RouteConfigurationResponseState(0); + EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); + EXPECT_EQ(response_state.error_message, + "Path not in the required format of /service/method"); } // Tests that LDS client should send a NACK if route match has path @@ -2456,7 +2495,9 @@ TEST_P(LdsRdsTest, RouteMatchHasInvalidPathMissingService) { SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); - EXPECT_EQ(RouteConfigurationResponseState(0), AdsServiceImpl::NACKED); + const auto& response_state = RouteConfigurationResponseState(0); + EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); + EXPECT_EQ(response_state.error_message, "Path contains empty service name"); } // Tests that LDS client should send a NACK if route match has path @@ -2477,7 +2518,9 @@ TEST_P(LdsRdsTest, RouteMatchHasInvalidPathMissingMethod) { SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); - EXPECT_EQ(RouteConfigurationResponseState(0), AdsServiceImpl::NACKED); + const auto& response_state = RouteConfigurationResponseState(0); + EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); + EXPECT_EQ(response_state.error_message, "Path contains empty method name"); } // Tests that LDS client should send a NACK if route has an action other than @@ -2490,12 +2533,11 @@ TEST_P(LdsRdsTest, RouteHasNoRouteAction) { SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); - EXPECT_EQ(RouteConfigurationResponseState(0), AdsServiceImpl::NACKED); + const auto& response_state = RouteConfigurationResponseState(0); + EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); + EXPECT_EQ(response_state.error_message, "No RouteAction found in route."); } -// TODO@donnadionne: Add more invalid config tests to cover all errors in -// xds_api.cc - // Tests that LDS client should send a NACK if RouteAction has a // cluster_specifier other than cluster in the LDS response. TEST_P(LdsRdsTest, RouteActionHasNoCluster) { @@ -2509,7 +2551,9 @@ TEST_P(LdsRdsTest, RouteActionHasNoCluster) { SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); - EXPECT_EQ(RouteConfigurationResponseState(0), AdsServiceImpl::NACKED); + const auto& response_state = RouteConfigurationResponseState(0); + EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); + EXPECT_EQ(response_state.error_message, "No cluster found in RouteAction."); } // Tests that LDS client times out when no response received. @@ -2685,8 +2729,8 @@ TEST_P(CdsTest, Vanilla) { SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); (void)SendRpc(); - EXPECT_EQ(balancers_[0]->ads_service()->cds_response_state(), - AdsServiceImpl::ACKED); + EXPECT_EQ(balancers_[0]->ads_service()->cds_response_state().state, + AdsServiceImpl::ResponseState::ACKED); } // Tests that CDS client should send a NACK if the cluster type in CDS response @@ -2698,8 +2742,10 @@ TEST_P(CdsTest, WrongClusterType) { SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); - EXPECT_EQ(balancers_[0]->ads_service()->cds_response_state(), - AdsServiceImpl::NACKED); + const auto& response_state = + balancers_[0]->ads_service()->cds_response_state(); + EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); + EXPECT_EQ(response_state.error_message, "DiscoveryType is not EDS."); } // Tests that CDS client should send a NACK if the eds_config in CDS response is @@ -2711,8 +2757,10 @@ TEST_P(CdsTest, WrongEdsConfig) { SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); - EXPECT_EQ(balancers_[0]->ads_service()->cds_response_state(), - AdsServiceImpl::NACKED); + const auto& response_state = + balancers_[0]->ads_service()->cds_response_state(); + EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); + EXPECT_EQ(response_state.error_message, "EDS ConfigSource is not ADS."); } // Tests that CDS client should send a NACK if the lb_policy in CDS response is @@ -2724,8 +2772,10 @@ TEST_P(CdsTest, WrongLbPolicy) { SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); - EXPECT_EQ(balancers_[0]->ads_service()->cds_response_state(), - AdsServiceImpl::NACKED); + const auto& response_state = + balancers_[0]->ads_service()->cds_response_state(); + EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); + EXPECT_EQ(response_state.error_message, "LB policy is not ROUND_ROBIN."); } // Tests that CDS client should send a NACK if the lrs_server in CDS response is @@ -2737,8 +2787,10 @@ TEST_P(CdsTest, WrongLrsServer) { SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); - EXPECT_EQ(balancers_[0]->ads_service()->cds_response_state(), - AdsServiceImpl::NACKED); + const auto& response_state = + balancers_[0]->ads_service()->cds_response_state(); + EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); + EXPECT_EQ(response_state.error_message, "LRS ConfigSource is not self."); } // Tests that CDS client times out when no response received. @@ -2771,8 +2823,11 @@ TEST_P(EdsTest, NacksSparsePriorityList) { balancers_[0]->ads_service()->SetEdsResource( AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); CheckRpcSendFailure(); - EXPECT_EQ(balancers_[0]->ads_service()->eds_response_state(), - AdsServiceImpl::NACKED); + const auto& response_state = + balancers_[0]->ads_service()->eds_response_state(); + EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); + EXPECT_EQ(response_state.error_message, + "EDS update includes sparse priority list"); } using LocalityMapTest = BasicTest; @@ -3221,8 +3276,8 @@ TEST_P(FailoverTest, MoveAllLocalitiesInCurrentPriorityToHigherPriority) { // When backend 3 gets traffic, we know the second update has been seen. WaitForBackend(3); // The ADS service of balancer 0 got at least 1 response. - EXPECT_GT(balancers_[0]->ads_service()->eds_response_state(), - AdsServiceImpl::NOT_SENT); + EXPECT_GT(balancers_[0]->ads_service()->eds_response_state().state, + AdsServiceImpl::ResponseState::NOT_SENT); delayed_resource_setter.join(); } @@ -3495,12 +3550,12 @@ TEST_P(BalancerUpdateTest, UpdateBalancersButKeepUsingOriginalBalancer) { // All 10 requests should have gone to the first backend. EXPECT_EQ(10U, backends_[0]->backend_service()->request_count()); // The ADS service of balancer 0 sent at least 1 response. - EXPECT_GT(balancers_[0]->ads_service()->eds_response_state(), - AdsServiceImpl::NOT_SENT); - EXPECT_EQ(balancers_[1]->ads_service()->eds_response_state(), - AdsServiceImpl::NOT_SENT); - EXPECT_EQ(balancers_[2]->ads_service()->eds_response_state(), - AdsServiceImpl::NOT_SENT); + EXPECT_GT(balancers_[0]->ads_service()->eds_response_state().state, + AdsServiceImpl::ResponseState::NOT_SENT); + EXPECT_EQ(balancers_[1]->ads_service()->eds_response_state().state, + AdsServiceImpl::ResponseState::NOT_SENT); + EXPECT_EQ(balancers_[2]->ads_service()->eds_response_state().state, + AdsServiceImpl::ResponseState::NOT_SENT); gpr_log(GPR_INFO, "========= ABOUT TO UPDATE 1 =========="); SetNextResolutionForLbChannel({balancers_[1]->port()}); gpr_log(GPR_INFO, "========= UPDATE 1 DONE =========="); @@ -3515,12 +3570,12 @@ TEST_P(BalancerUpdateTest, UpdateBalancersButKeepUsingOriginalBalancer) { // first balancer, which doesn't assign the second backend. EXPECT_EQ(0U, backends_[1]->backend_service()->request_count()); // The ADS service of balancer 0 sent at least 1 response. - EXPECT_GT(balancers_[0]->ads_service()->eds_response_state(), - AdsServiceImpl::NOT_SENT); - EXPECT_EQ(balancers_[1]->ads_service()->eds_response_state(), - AdsServiceImpl::NOT_SENT); - EXPECT_EQ(balancers_[2]->ads_service()->eds_response_state(), - AdsServiceImpl::NOT_SENT); + EXPECT_GT(balancers_[0]->ads_service()->eds_response_state().state, + AdsServiceImpl::ResponseState::NOT_SENT); + EXPECT_EQ(balancers_[1]->ads_service()->eds_response_state().state, + AdsServiceImpl::ResponseState::NOT_SENT); + EXPECT_EQ(balancers_[2]->ads_service()->eds_response_state().state, + AdsServiceImpl::ResponseState::NOT_SENT); } // Tests that the old LB call is still used after multiple balancer address @@ -3550,12 +3605,12 @@ TEST_P(BalancerUpdateTest, Repeated) { // All 10 requests should have gone to the first backend. EXPECT_EQ(10U, backends_[0]->backend_service()->request_count()); // The ADS service of balancer 0 sent at least 1 response. - EXPECT_GT(balancers_[0]->ads_service()->eds_response_state(), - AdsServiceImpl::NOT_SENT); - EXPECT_EQ(balancers_[1]->ads_service()->eds_response_state(), - AdsServiceImpl::NOT_SENT); - EXPECT_EQ(balancers_[2]->ads_service()->eds_response_state(), - AdsServiceImpl::NOT_SENT); + EXPECT_GT(balancers_[0]->ads_service()->eds_response_state().state, + AdsServiceImpl::ResponseState::NOT_SENT); + EXPECT_EQ(balancers_[1]->ads_service()->eds_response_state().state, + AdsServiceImpl::ResponseState::NOT_SENT); + EXPECT_EQ(balancers_[2]->ads_service()->eds_response_state().state, + AdsServiceImpl::ResponseState::NOT_SENT); std::vector ports; ports.emplace_back(balancers_[0]->port()); ports.emplace_back(balancers_[1]->port()); @@ -3614,12 +3669,12 @@ TEST_P(BalancerUpdateTest, DeadUpdate) { // All 10 requests should have gone to the first backend. EXPECT_EQ(10U, backends_[0]->backend_service()->request_count()); // The ADS service of balancer 0 sent at least 1 response. - EXPECT_GT(balancers_[0]->ads_service()->eds_response_state(), - AdsServiceImpl::NOT_SENT); - EXPECT_EQ(balancers_[1]->ads_service()->eds_response_state(), - AdsServiceImpl::NOT_SENT); - EXPECT_EQ(balancers_[2]->ads_service()->eds_response_state(), - AdsServiceImpl::NOT_SENT); + EXPECT_GT(balancers_[0]->ads_service()->eds_response_state().state, + AdsServiceImpl::ResponseState::NOT_SENT); + EXPECT_EQ(balancers_[1]->ads_service()->eds_response_state().state, + AdsServiceImpl::ResponseState::NOT_SENT); + EXPECT_EQ(balancers_[2]->ads_service()->eds_response_state().state, + AdsServiceImpl::ResponseState::NOT_SENT); // Kill balancer 0 gpr_log(GPR_INFO, "********** ABOUT TO KILL BALANCER 0 *************"); balancers_[0]->Shutdown(); @@ -3632,12 +3687,12 @@ TEST_P(BalancerUpdateTest, DeadUpdate) { EXPECT_EQ(20U, backends_[0]->backend_service()->request_count()); EXPECT_EQ(0U, backends_[1]->backend_service()->request_count()); // The ADS service of no balancers sent anything - EXPECT_EQ(balancers_[0]->ads_service()->eds_response_state(), - AdsServiceImpl::NOT_SENT); - EXPECT_EQ(balancers_[1]->ads_service()->eds_response_state(), - AdsServiceImpl::NOT_SENT); - EXPECT_EQ(balancers_[2]->ads_service()->eds_response_state(), - AdsServiceImpl::NOT_SENT); + EXPECT_EQ(balancers_[0]->ads_service()->eds_response_state().state, + AdsServiceImpl::ResponseState::NOT_SENT); + EXPECT_EQ(balancers_[1]->ads_service()->eds_response_state().state, + AdsServiceImpl::ResponseState::NOT_SENT); + EXPECT_EQ(balancers_[2]->ads_service()->eds_response_state().state, + AdsServiceImpl::ResponseState::NOT_SENT); gpr_log(GPR_INFO, "========= ABOUT TO UPDATE 1 =========="); SetNextResolutionForLbChannel({balancers_[1]->port()}); gpr_log(GPR_INFO, "========= UPDATE 1 DONE =========="); @@ -3654,12 +3709,12 @@ TEST_P(BalancerUpdateTest, DeadUpdate) { // All 10 requests should have gone to the second backend. EXPECT_EQ(10U, backends_[1]->backend_service()->request_count()); // The ADS service of balancer 1 sent at least 1 response. - EXPECT_EQ(balancers_[0]->ads_service()->eds_response_state(), - AdsServiceImpl::NOT_SENT); - EXPECT_GT(balancers_[1]->ads_service()->eds_response_state(), - AdsServiceImpl::NOT_SENT); - EXPECT_EQ(balancers_[2]->ads_service()->eds_response_state(), - AdsServiceImpl::NOT_SENT); + EXPECT_EQ(balancers_[0]->ads_service()->eds_response_state().state, + AdsServiceImpl::ResponseState::NOT_SENT); + EXPECT_GT(balancers_[1]->ads_service()->eds_response_state().state, + AdsServiceImpl::ResponseState::NOT_SENT); + EXPECT_EQ(balancers_[2]->ads_service()->eds_response_state().state, + AdsServiceImpl::ResponseState::NOT_SENT); } // The re-resolution tests are deferred because they rely on the fallback mode, From f6e6b9f64032a2ba9119acb120dd2c2f30391446 Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Thu, 30 Apr 2020 15:14:54 -0700 Subject: [PATCH 39/47] Fix the pylint complain --- examples/python/data_transmission/alts_client.py | 11 ++++++----- examples/python/data_transmission/alts_server.py | 5 +++-- src/python/grpcio/grpc/__init__.py | 4 ++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/examples/python/data_transmission/alts_client.py b/examples/python/data_transmission/alts_client.py index d196dbd17c2..eb328667cd6 100644 --- a/examples/python/data_transmission/alts_client.py +++ b/examples/python/data_transmission/alts_client.py @@ -17,8 +17,9 @@ The example would only successfully run in GCP environment.""" import grpc -import client import demo_pb2_grpc +from client import (bidirectional_streaming_method, client_streaming_method, + server_streaming_method, simple_method) SERVER_ADDRESS = "localhost:23333" @@ -28,10 +29,10 @@ def main(): SERVER_ADDRESS, credentials=grpc.alts_channel_credentials()) as channel: stub = demo_pb2_grpc.GRPCDemoStub(channel) - client.simple_method(stub) - client.client_streaming_method(stub) - client.server_streaming_method(stub) - client.bidirectional_streaming_method(stub) + simple_method(stub) + client_streaming_method(stub) + server_streaming_method(stub) + bidirectional_streaming_method(stub) if __name__ == '__main__': diff --git a/examples/python/data_transmission/alts_server.py b/examples/python/data_transmission/alts_server.py index 7f1d2f8983d..4e8747ce7a9 100644 --- a/examples/python/data_transmission/alts_server.py +++ b/examples/python/data_transmission/alts_server.py @@ -18,15 +18,16 @@ The example would only successfully run in GCP environment.""" from concurrent import futures import grpc + import demo_pb2_grpc -import server +from server import DemoServer SERVER_ADDRESS = 'localhost:23333' def main(): svr = grpc.server(futures.ThreadPoolExecutor()) - demo_pb2_grpc.add_GRPCDemoServicer_to_server(server.DemoServer(), svr) + demo_pb2_grpc.add_GRPCDemoServicer_to_server(DemoServer(), svr) svr.add_secure_port(SERVER_ADDRESS, server_credentials=grpc.alts_server_credentials()) print("------------------start Python GRPC server with ALTS encryption") diff --git a/src/python/grpcio/grpc/__init__.py b/src/python/grpcio/grpc/__init__.py index f4c261c23f2..303d89c6263 100644 --- a/src/python/grpcio/grpc/__init__.py +++ b/src/python/grpcio/grpc/__init__.py @@ -1833,7 +1833,7 @@ def local_server_credentials(local_connect_type=LocalConnectionType.LOCAL_TCP): _cygrpc.server_credentials_local(local_connect_type.value)) -def alts_channel_credentials(service_accounts=[]): +def alts_channel_credentials(service_accounts=None): """Creates a ChannelCredentials for use with an ALTS-enabled Channel. This is an EXPERIMENTAL API. @@ -1851,7 +1851,7 @@ def alts_channel_credentials(service_accounts=[]): A ChannelCredentials for use with an ALTS-enabled Channel """ return ChannelCredentials( - _cygrpc.channel_credentials_alts(service_accounts)) + _cygrpc.channel_credentials_alts(service_accounts or [])) def alts_server_credentials(): From 6a6481cb8378cec31bf043ded593d8c07c56b46d Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Thu, 30 Apr 2020 15:44:48 -0700 Subject: [PATCH 40/47] Reviewer comments --- include/grpcpp/impl/codegen/call_op_set.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/include/grpcpp/impl/codegen/call_op_set.h b/include/grpcpp/impl/codegen/call_op_set.h index 79fa08ad6dc..994b9998390 100644 --- a/include/grpcpp/impl/codegen/call_op_set.h +++ b/include/grpcpp/impl/codegen/call_op_set.h @@ -421,8 +421,6 @@ Status CallOpSendMessage::SendMessagePtr(const M* message) { template class CallOpRecvMessage { public: - CallOpRecvMessage() {} - void RecvMessage(R* message) { message_ = message; } // Do not change status if no message is received. @@ -525,8 +523,6 @@ class DeserializeFuncType final : public DeserializeFunc { class CallOpGenericRecvMessage { public: - CallOpGenericRecvMessage() {} - template void RecvMessage(R* message) { // Use an explicit base class pointer to avoid resolution error in the From 88e43d9901ebb0c948c3cdef900e397c41cc41c8 Mon Sep 17 00:00:00 2001 From: yang-g Date: Thu, 30 Apr 2020 15:46:47 -0700 Subject: [PATCH 41/47] Adjust wait timeout according to test slowdown factor --- test/core/util/test_config.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/core/util/test_config.cc b/test/core/util/test_config.cc index 115a9833c44..c6ee9fc2400 100644 --- a/test/core/util/test_config.cc +++ b/test/core/util/test_config.cc @@ -18,6 +18,7 @@ #include "test/core/util/test_config.h" +#include #include #include #include @@ -402,13 +403,12 @@ TestEnvironment::TestEnvironment(int argc, char** argv) { TestEnvironment::~TestEnvironment() { // This will wait until gRPC shutdown has actually happened to make sure // no gRPC resources (such as thread) are active. (timeout = 10s) - gpr_timespec timeout = gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), - gpr_time_from_seconds(10, GPR_TIMESPAN)); + gpr_timespec deadline = grpc_timeout_seconds_to_deadline(10); while (grpc_is_initialized()) { grpc_maybe_wait_for_async_shutdown(); gpr_sleep_until(gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), gpr_time_from_millis(1, GPR_TIMESPAN))); - if (gpr_time_cmp(gpr_now(GPR_CLOCK_REALTIME), timeout) > 0) { + if (gpr_time_cmp(gpr_now(GPR_CLOCK_MONOTONIC), deadline) > 0) { gpr_log(GPR_ERROR, "Timeout in waiting for gRPC shutdown"); break; } From 72a42151edcf6d60da129d8baa264281fbeda137 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Fri, 1 May 2020 07:23:01 -0700 Subject: [PATCH 42/47] xds: Check RDS ConfigSource in LDS response. --- .../ext/filters/client_channel/xds/xds_api.cc | 14 +++++- test/cpp/end2end/xds_end2end_test.cc | 46 ++++++++++++++++++- 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/src/core/ext/filters/client_channel/xds/xds_api.cc b/src/core/ext/filters/client_channel/xds/xds_api.cc index 60ca930e060..f124052ce84 100644 --- a/src/core/ext/filters/client_channel/xds/xds_api.cc +++ b/src/core/ext/filters/client_channel/xds/xds_api.cc @@ -1174,10 +1174,22 @@ grpc_error* LdsResponseParse(XdsClient* client, TraceFlag* tracer, return GRPC_ERROR_CREATE_FROM_STATIC_STRING( "HttpConnectionManager neither has inlined route_config nor RDS."); } - // Get the route_config_name. const envoy_config_filter_network_http_connection_manager_v2_Rds* rds = envoy_config_filter_network_http_connection_manager_v2_HttpConnectionManager_rds( http_connection_manager); + // Check that the ConfigSource specifies ADS. + const envoy_api_v2_core_ConfigSource* config_source = + envoy_config_filter_network_http_connection_manager_v2_Rds_config_source( + rds); + if (config_source == nullptr) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "HttpConnectionManager missing config_source for RDS."); + } + if (!envoy_api_v2_core_ConfigSource_has_ads(config_source)) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "HttpConnectionManager ConfigSource for RDS does not specify ADS."); + } + // Get the route_config_name. const upb_strview route_config_name = envoy_config_filter_network_http_connection_manager_v2_Rds_route_config_name( rds); diff --git a/test/cpp/end2end/xds_end2end_test.cc b/test/cpp/end2end/xds_end2end_test.cc index 5fb6a62b19d..7d70feadbad 100644 --- a/test/cpp/end2end/xds_end2end_test.cc +++ b/test/cpp/end2end/xds_end2end_test.cc @@ -713,8 +713,9 @@ class AdsServiceImpl : public AggregatedDiscoveryService::Service, void SetLdsToUseDynamicRds() { auto listener = default_listener_; HttpConnectionManager http_connection_manager; - http_connection_manager.mutable_rds()->set_route_config_name( - kDefaultResourceName); + auto* rds = http_connection_manager.mutable_rds(); + rds->set_route_config_name(kDefaultResourceName); + rds->mutable_config_source()->mutable_ads(); listener.mutable_api_listener()->mutable_api_listener()->PackFrom( http_connection_manager); SetLdsResource(listener, kDefaultResourceName); @@ -2191,6 +2192,47 @@ TEST_P(LdsTest, WrongRouteSpecifier) { "HttpConnectionManager neither has inlined route_config nor RDS."); } +// Tests that LDS client should send a NACK if the rds message in the +// http_connection_manager is missing the config_source field. +TEST_P(LdsTest, RdsMissingConfigSource) { + auto listener = balancers_[0]->ads_service()->default_listener(); + HttpConnectionManager http_connection_manager; + http_connection_manager.mutable_rds()->set_route_config_name( + kDefaultResourceName); + listener.mutable_api_listener()->mutable_api_listener()->PackFrom( + http_connection_manager); + balancers_[0]->ads_service()->SetLdsResource(listener, kDefaultResourceName); + SetNextResolution({}); + SetNextResolutionForLbChannelAllBalancers(); + CheckRpcSendFailure(); + const auto& response_state = + balancers_[0]->ads_service()->lds_response_state(); + EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); + EXPECT_EQ(response_state.error_message, + "HttpConnectionManager missing config_source for RDS."); +} + +// Tests that LDS client should send a NACK if the rds message in the +// http_connection_manager has a config_source field that does not specify ADS. +TEST_P(LdsTest, RdsConfigSourceDoesNotSpecifyAds) { + auto listener = balancers_[0]->ads_service()->default_listener(); + HttpConnectionManager http_connection_manager; + auto* rds = http_connection_manager.mutable_rds(); + rds->set_route_config_name(kDefaultResourceName); + rds->mutable_config_source()->mutable_self(); + listener.mutable_api_listener()->mutable_api_listener()->PackFrom( + http_connection_manager); + balancers_[0]->ads_service()->SetLdsResource(listener, kDefaultResourceName); + SetNextResolution({}); + SetNextResolutionForLbChannelAllBalancers(); + CheckRpcSendFailure(); + const auto& response_state = + balancers_[0]->ads_service()->lds_response_state(); + EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); + EXPECT_EQ(response_state.error_message, + "HttpConnectionManager ConfigSource for RDS does not specify ADS."); +} + using LdsRdsTest = BasicTest; // Tests that LDS client should send an ACK upon correct LDS response (with From eb2b24ce25a134beea04a0fdc5593539e7a1a7d9 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Fri, 1 May 2020 16:21:42 -0700 Subject: [PATCH 43/47] Patch typo --- src/core/lib/iomgr/iomgr_posix_cfstream.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/iomgr_posix_cfstream.cc b/src/core/lib/iomgr/iomgr_posix_cfstream.cc index 24f9f4101b2..30c44edb5d6 100644 --- a/src/core/lib/iomgr/iomgr_posix_cfstream.cc +++ b/src/core/lib/iomgr/iomgr_posix_cfstream.cc @@ -25,7 +25,7 @@ /// platforms), the users can disable CFStream with environment variable /// "grpc_cfstream=0". This will let gRPC to fallback to use POSIX sockets. In /// addition, the users may choose to use an alternative CFRunLoop based pollset -/// "ev_apple" by setting environment variable "grpc_cfstream_run_loop=1". This +/// "ev_apple" by setting environment variable "GRPC_CFSTREAM_RUN_LOOP=1". This /// pollset resolves a bug from Apple when CFStream streams dispatch events to /// dispatch queues. The caveat of this pollset is that users may not be able to /// run a gRPC server in the same process. From 9f6ce14dd97fa537fb9c0c251b46fb40d4471edd Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Fri, 1 May 2020 16:30:39 -0700 Subject: [PATCH 44/47] Update CFStream doc with CFRunLoop poller info --- src/objective-c/README-CFSTREAM.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/objective-c/README-CFSTREAM.md b/src/objective-c/README-CFSTREAM.md index d4d26828534..7d75d47a606 100644 --- a/src/objective-c/README-CFSTREAM.md +++ b/src/objective-c/README-CFSTREAM.md @@ -23,8 +23,18 @@ platforms, you can turn it on with macro `GRPC_CFSTREAM=1` for the pod 'gRPC-Cor case of problem and you want to disable CFStream on iOS, you can set environment variable "grpc\_cfstream=0". +## Caveats +It is known to us that the CFStream API has some bug which will cause gRPC's CFStream networking +layer to stall occasionally. The issue mostly occur on MacOS systems (including iOS simulators on +MacOS); iOS may be affected too but we have not seen issue there. gRPC provides a workaround to this +problem with an alternative poller based on CFRunLoop. The poller can be enabled by setting +environment variable `GRPC_CFSTREAM_RUN_LOOP=1`. Note that the poller is a client side only poller +that does not support running a server on it. That means if an app opts in to the CFRunLoop-based +poller, the app cannot host a gRPC server (gRPC Objective-C does not support running a server but +other languages running on iOS do support it). + ## Notes - Currently we do not support platforms other than iOS, although it is likely that this integration can run on MacOS targets with Apple's compiler. -- Let us know if you meet any issue by filing issue and ping @muxi. +- Let us know if you meet any issue by filing issue and ping @stanleycheung. From 5557f80ac41ea6ef4f1542009280346c6ed949cd Mon Sep 17 00:00:00 2001 From: Donna Dionne Date: Mon, 4 May 2020 10:36:27 -0700 Subject: [PATCH 45/47] Passing C repo manager to markdroth --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- .github/ISSUE_TEMPLATE/cleanup_request.md | 2 +- .github/ISSUE_TEMPLATE/feature_request.md | 2 +- .github/pull_request_template.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 680cd706703..9011a669d3d 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -2,7 +2,7 @@ name: Report a bug about: Create a report to help us improve labels: kind/bug, priority/P2 -assignees: donnadionne +assignees: markdroth --- diff --git a/.github/ISSUE_TEMPLATE/cleanup_request.md b/.github/ISSUE_TEMPLATE/cleanup_request.md index 96c8bda3c4c..9cf36e6fa0b 100644 --- a/.github/ISSUE_TEMPLATE/cleanup_request.md +++ b/.github/ISSUE_TEMPLATE/cleanup_request.md @@ -2,7 +2,7 @@ name: Request a cleanup about: Suggest a cleanup in our repository labels: kind/internal cleanup, priority/P2 -assignees: donnadionne +assignees: markdroth --- diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index a7b05e99004..0d9906a9eb0 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -2,7 +2,7 @@ name: Request a feature about: Suggest an idea for this project labels: kind/enhancement, priority/P2 -assignees: donnadionne +assignees: markdroth --- diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index eddba6676da..a85cfad9c7a 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -8,4 +8,4 @@ If you know who should review your pull request, please remove the mentioning be --> -@donnadionne +@markdroth From 9640aa3db80b20e131b4011f63da27dad79730fb Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Mon, 4 May 2020 16:55:33 -0700 Subject: [PATCH 46/47] Add FB number in the comment --- src/objective-c/README-CFSTREAM.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/objective-c/README-CFSTREAM.md b/src/objective-c/README-CFSTREAM.md index 7d75d47a606..94a43ff3d06 100644 --- a/src/objective-c/README-CFSTREAM.md +++ b/src/objective-c/README-CFSTREAM.md @@ -24,14 +24,14 @@ case of problem and you want to disable CFStream on iOS, you can set environment "grpc\_cfstream=0". ## Caveats -It is known to us that the CFStream API has some bug which will cause gRPC's CFStream networking -layer to stall occasionally. The issue mostly occur on MacOS systems (including iOS simulators on -MacOS); iOS may be affected too but we have not seen issue there. gRPC provides a workaround to this -problem with an alternative poller based on CFRunLoop. The poller can be enabled by setting -environment variable `GRPC_CFSTREAM_RUN_LOOP=1`. Note that the poller is a client side only poller -that does not support running a server on it. That means if an app opts in to the CFRunLoop-based -poller, the app cannot host a gRPC server (gRPC Objective-C does not support running a server but -other languages running on iOS do support it). +It is known to us that the CFStream API has some bug (FB6162039) which will cause gRPC's CFStream +networking layer to stall occasionally. The issue mostly occur on MacOS systems (including iOS +simulators on MacOS); iOS may be affected too but we have not seen issue there. gRPC provides a +workaround to this problem with an alternative poller based on CFRunLoop. The poller can be enabled +by setting environment variable `GRPC_CFSTREAM_RUN_LOOP=1`. Note that the poller is a client side +only poller that does not support running a server on it. That means if an app opts in to the +CFRunLoop-based poller, the app cannot host a gRPC server (gRPC Objective-C does not support running +a server but other languages running on iOS do support it). ## Notes From ac502892740dde352f70b132010d24f050565534 Mon Sep 17 00:00:00 2001 From: Donna Dionne Date: Mon, 4 May 2020 17:14:54 -0700 Subject: [PATCH 47/47] Modify the signature of all SetXdsResource methods to avoid un-necessary parameter "name". --- test/cpp/end2end/xds_end2end_test.cc | 222 +++++++++++++-------------- 1 file changed, 105 insertions(+), 117 deletions(-) diff --git a/test/cpp/end2end/xds_end2end_test.cc b/test/cpp/end2end/xds_end2end_test.cc index 7d70feadbad..f2dc6f08e0e 100644 --- a/test/cpp/end2end/xds_end2end_test.cc +++ b/test/cpp/end2end/xds_end2end_test.cc @@ -420,10 +420,10 @@ class AdsServiceImpl : public AggregatedDiscoveryService::Service, auto* route = virtual_host->add_routes(); route->mutable_match()->set_prefix(""); route->mutable_route()->set_cluster(kDefaultResourceName); - SetRdsResource(default_route_config_, kDefaultResourceName); + SetRdsResource(default_route_config_); // Construct LDS response data (with inlined RDS result). default_listener_ = BuildListener(default_route_config_); - SetLdsResource(default_listener_, kDefaultResourceName); + SetLdsResource(default_listener_); // Construct CDS response data. default_cluster_.set_name(kDefaultResourceName); default_cluster_.set_type(envoy::api::v2::Cluster::EDS); @@ -434,7 +434,7 @@ class AdsServiceImpl : public AggregatedDiscoveryService::Service, if (enable_load_reporting) { default_cluster_.mutable_lrs_server()->mutable_self(); } - SetCdsResource(default_cluster_, kDefaultResourceName); + SetCdsResource(default_cluster_); } Status StreamAggregatedResources(ServerContext* context, @@ -684,30 +684,28 @@ class AdsServiceImpl : public AggregatedDiscoveryService::Service, } } - void SetLdsResource(const Listener& listener, const std::string& name) { + void SetLdsResource(const Listener& listener) { google::protobuf::Any resource; resource.PackFrom(listener); - SetResource(std::move(resource), kLdsTypeUrl, name); + SetResource(std::move(resource), kLdsTypeUrl, listener.name()); } - void SetRdsResource(const RouteConfiguration& route, - const std::string& name) { + void SetRdsResource(const RouteConfiguration& route) { google::protobuf::Any resource; resource.PackFrom(route); - SetResource(std::move(resource), kRdsTypeUrl, name); + SetResource(std::move(resource), kRdsTypeUrl, route.name()); } - void SetCdsResource(const Cluster& cluster, const std::string& name) { + void SetCdsResource(const Cluster& cluster) { google::protobuf::Any resource; resource.PackFrom(cluster); - SetResource(std::move(resource), kCdsTypeUrl, name); + SetResource(std::move(resource), kCdsTypeUrl, cluster.name()); } - void SetEdsResource(const ClusterLoadAssignment& assignment, - const std::string& name) { + void SetEdsResource(const ClusterLoadAssignment& assignment) { google::protobuf::Any resource; resource.PackFrom(assignment); - SetResource(std::move(resource), kEdsTypeUrl, name); + SetResource(std::move(resource), kEdsTypeUrl, assignment.cluster_name()); } void SetLdsToUseDynamicRds() { @@ -718,7 +716,7 @@ class AdsServiceImpl : public AggregatedDiscoveryService::Service, rds->mutable_config_source()->mutable_ads(); listener.mutable_api_listener()->mutable_api_listener()->PackFrom( http_connection_manager); - SetLdsResource(listener, kDefaultResourceName); + SetLdsResource(listener); } static Listener BuildListener(const RouteConfiguration& route_config) { @@ -1482,11 +1480,10 @@ class XdsEnd2endTest : public ::testing::TestWithParam { void SetRouteConfiguration(int idx, const RouteConfiguration& route_config) { if (GetParam().enable_rds_testing()) { - balancers_[idx]->ads_service()->SetRdsResource(route_config, - kDefaultResourceName); + balancers_[idx]->ads_service()->SetRdsResource(route_config); } else { balancers_[idx]->ads_service()->SetLdsResource( - AdsServiceImpl::BuildListener(route_config), kDefaultResourceName); + AdsServiceImpl::BuildListener(route_config)); } } @@ -1503,10 +1500,10 @@ class XdsEnd2endTest : public ::testing::TestWithParam { // via bind with a qualified name, it needs to be public. void SetEdsResourceWithDelay(size_t i, const ClusterLoadAssignment& assignment, - int delay_ms, const std::string& name) { + int delay_ms) { GPR_ASSERT(delay_ms > 0); gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(delay_ms)); - balancers_[i]->ads_service()->SetEdsResource(assignment, name); + balancers_[i]->ads_service()->SetEdsResource(assignment); } protected: @@ -1697,7 +1694,7 @@ TEST_P(BasicTest, Vanilla) { {"locality0", GetBackendPorts()}, }); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); // Make sure that trying to connect works without a call. channel_->GetState(true /* try_to_connect */); // We need to wait for all backends to come online. @@ -1727,7 +1724,7 @@ TEST_P(BasicTest, IgnoresUnhealthyEndpoints) { {envoy::api::v2::HealthStatus::DRAINING}}, }); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); // Make sure that trying to connect works without a call. channel_->GetState(true /* try_to_connect */); // We need to wait for all backends to come online. @@ -1753,7 +1750,7 @@ TEST_P(BasicTest, SameBackendListedMultipleTimes) { }); const size_t kNumRpcsPerAddress = 10; balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); // We need to wait for the backend to come online. WaitForBackend(0); // Send kNumRpcsPerAddress RPCs per server. @@ -1778,15 +1775,14 @@ TEST_P(BasicTest, InitiallyEmptyServerlist) { empty_locality, }); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); // Send non-empty serverlist only after kServerlistDelayMs. args = AdsServiceImpl::EdsResourceArgs({ {"locality0", GetBackendPorts()}, }); std::thread delayed_resource_setter( std::bind(&BasicTest::SetEdsResourceWithDelay, this, 0, - AdsServiceImpl::BuildEdsResource(args), kServerlistDelayMs, - kDefaultResourceName)); + AdsServiceImpl::BuildEdsResource(args), kServerlistDelayMs)); const auto t0 = system_clock::now(); // Client will block: LB will initially send empty serverlist. CheckRpcSendOk( @@ -1816,7 +1812,7 @@ TEST_P(BasicTest, AllServersUnreachableFailFast) { {"locality0", ports}, }); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); const Status status = SendRpc(); // The error shouldn't be DEADLINE_EXCEEDED. EXPECT_EQ(StatusCode::UNAVAILABLE, status.error_code()); @@ -1831,7 +1827,7 @@ TEST_P(BasicTest, BackendsRestart) { {"locality0", GetBackendPorts()}, }); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); WaitForAllBackends(); // Stop backends. RPCs should fail. ShutdownAllBackends(); @@ -1861,7 +1857,7 @@ TEST_P(XdsResolverOnlyTest, ChangeClusters) { {"locality0", GetBackendPorts(0, 2)}, }); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); // We need to wait for all backends to come online. WaitForAllBackends(0, 2); // Populate new EDS resource. @@ -1869,12 +1865,11 @@ TEST_P(XdsResolverOnlyTest, ChangeClusters) { {"locality0", GetBackendPorts(2, 4)}, }); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args2, kNewClusterName), - kNewClusterName); + AdsServiceImpl::BuildEdsResource(args2, kNewClusterName)); // Populate new CDS resource. Cluster new_cluster = balancers_[0]->ads_service()->default_cluster(); new_cluster.set_name(kNewClusterName); - balancers_[0]->ads_service()->SetCdsResource(new_cluster, kNewClusterName); + balancers_[0]->ads_service()->SetCdsResource(new_cluster); // Change RDS resource to point to new cluster. RouteConfiguration new_route_config = balancers_[0]->ads_service()->default_route_config(); @@ -1884,7 +1879,7 @@ TEST_P(XdsResolverOnlyTest, ChangeClusters) { ->set_cluster(kNewClusterName); Listener listener = balancers_[0]->ads_service()->BuildListener(new_route_config); - balancers_[0]->ads_service()->SetLdsResource(listener, kDefaultResourceName); + balancers_[0]->ads_service()->SetLdsResource(listener); // Wait for all new backends to be used. std::tuple counts = WaitForAllBackends(2, 4); // Make sure no RPCs failed in the transition. @@ -1899,7 +1894,7 @@ TEST_P(XdsResolverOnlyTest, ClusterRemoved) { {"locality0", GetBackendPorts()}, }); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); // We need to wait for all backends to come online. WaitForAllBackends(); // Unset CDS resource. @@ -1922,7 +1917,7 @@ TEST_P(XdsResolverOnlyTest, RestartsRequestsUponReconnection) { {"locality0", GetBackendPorts(0, 2)}, }); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); // We need to wait for all backends to come online. WaitForAllBackends(0, 2); // Now shut down and restart the balancer. When the client @@ -1937,12 +1932,11 @@ TEST_P(XdsResolverOnlyTest, RestartsRequestsUponReconnection) { {"locality0", GetBackendPorts(2, 4)}, }); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args2, kNewClusterName), - kNewClusterName); + AdsServiceImpl::BuildEdsResource(args2, kNewClusterName)); // Populate new CDS resource. Cluster new_cluster = balancers_[0]->ads_service()->default_cluster(); new_cluster.set_name(kNewClusterName); - balancers_[0]->ads_service()->SetCdsResource(new_cluster, kNewClusterName); + balancers_[0]->ads_service()->SetCdsResource(new_cluster); // Change RDS resource to point to new cluster. RouteConfiguration new_route_config = balancers_[0]->ads_service()->default_route_config(); @@ -1950,8 +1944,7 @@ TEST_P(XdsResolverOnlyTest, RestartsRequestsUponReconnection) { ->mutable_routes(0) ->mutable_route() ->set_cluster(kNewClusterName); - balancers_[0]->ads_service()->SetRdsResource(new_route_config, - kDefaultResourceName); + balancers_[0]->ads_service()->SetRdsResource(new_route_config); // Wait for all new backends to be used. std::tuple counts = WaitForAllBackends(2, 4); // Make sure no RPCs failed in the transition. @@ -1966,14 +1959,14 @@ TEST_P(XdsResolverOnlyTest, DefaultRouteSpecifiesSlashPrefix) { ->mutable_match() ->set_prefix("/"); balancers_[0]->ads_service()->SetLdsResource( - AdsServiceImpl::BuildListener(route_config), kDefaultResourceName); + AdsServiceImpl::BuildListener(route_config)); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); AdsServiceImpl::EdsResourceArgs args({ {"locality0", GetBackendPorts()}, }); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); // We need to wait for all backends to come online. WaitForAllBackends(); } @@ -1995,18 +1988,17 @@ TEST_P(XdsResolverLoadReportingOnlyTest, ChangeClusters) { {"locality0", GetBackendPorts(0, 2)}, }); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); // cluster kNewClusterName -> locality1 -> backends 2 and 3 AdsServiceImpl::EdsResourceArgs args2({ {"locality1", GetBackendPorts(2, 4)}, }); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args2, kNewClusterName), - kNewClusterName); + AdsServiceImpl::BuildEdsResource(args2, kNewClusterName)); // CDS resource for kNewClusterName. Cluster new_cluster = balancers_[0]->ads_service()->default_cluster(); new_cluster.set_name(kNewClusterName); - balancers_[0]->ads_service()->SetCdsResource(new_cluster, kNewClusterName); + balancers_[0]->ads_service()->SetCdsResource(new_cluster); // Wait for all backends to come online. int num_ok = 0; int num_failure = 0; @@ -2047,7 +2039,7 @@ TEST_P(XdsResolverLoadReportingOnlyTest, ChangeClusters) { ->set_cluster(kNewClusterName); Listener listener = balancers_[0]->ads_service()->BuildListener(new_route_config); - balancers_[0]->ads_service()->SetLdsResource(listener, kDefaultResourceName); + balancers_[0]->ads_service()->SetLdsResource(listener); // Wait for all new backends to be used. std::tie(num_ok, num_failure, num_drops) = WaitForAllBackends(2, 4); // The load report received at the balancer should be correct. @@ -2126,7 +2118,7 @@ TEST_P(SecureNamingTest, TargetNameIsExpected) { {"locality0", GetBackendPorts()}, }); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); // Make sure that trying to connect works without a call. channel_->GetState(true /* try_to_connect */); // We need to wait for all backends to come online. @@ -2163,7 +2155,7 @@ using LdsTest = BasicTest; TEST_P(LdsTest, NoApiListener) { auto listener = balancers_[0]->ads_service()->default_listener(); listener.clear_api_listener(); - balancers_[0]->ads_service()->SetLdsResource(listener, kDefaultResourceName); + balancers_[0]->ads_service()->SetLdsResource(listener); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); @@ -2181,7 +2173,7 @@ TEST_P(LdsTest, WrongRouteSpecifier) { http_connection_manager.mutable_scoped_routes(); listener.mutable_api_listener()->mutable_api_listener()->PackFrom( http_connection_manager); - balancers_[0]->ads_service()->SetLdsResource(listener, kDefaultResourceName); + balancers_[0]->ads_service()->SetLdsResource(listener); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); @@ -2201,7 +2193,7 @@ TEST_P(LdsTest, RdsMissingConfigSource) { kDefaultResourceName); listener.mutable_api_listener()->mutable_api_listener()->PackFrom( http_connection_manager); - balancers_[0]->ads_service()->SetLdsResource(listener, kDefaultResourceName); + balancers_[0]->ads_service()->SetLdsResource(listener); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); @@ -2222,7 +2214,7 @@ TEST_P(LdsTest, RdsConfigSourceDoesNotSpecifyAds) { rds->mutable_config_source()->mutable_self(); listener.mutable_api_listener()->mutable_api_listener()->PackFrom( http_connection_manager); - balancers_[0]->ads_service()->SetLdsResource(listener, kDefaultResourceName); + balancers_[0]->ads_service()->SetLdsResource(listener); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); @@ -2636,20 +2628,18 @@ TEST_P(LdsRdsTest, XdsRoutingPathMatching) { {"locality0", GetBackendPorts(3, 4)}, }); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args1, kNewCluster1Name), - kNewCluster1Name); + AdsServiceImpl::BuildEdsResource(args1, kNewCluster1Name)); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args2, kNewCluster2Name), - kNewCluster2Name); + AdsServiceImpl::BuildEdsResource(args2, kNewCluster2Name)); // Populate new CDS resources. Cluster new_cluster1 = balancers_[0]->ads_service()->default_cluster(); new_cluster1.set_name(kNewCluster1Name); - balancers_[0]->ads_service()->SetCdsResource(new_cluster1, kNewCluster1Name); + balancers_[0]->ads_service()->SetCdsResource(new_cluster1); Cluster new_cluster2 = balancers_[0]->ads_service()->default_cluster(); new_cluster2.set_name(kNewCluster2Name); - balancers_[0]->ads_service()->SetCdsResource(new_cluster2, kNewCluster2Name); + balancers_[0]->ads_service()->SetCdsResource(new_cluster2); // Populating Route Configurations for LDS. RouteConfiguration new_route_config = balancers_[0]->ads_service()->default_route_config(); @@ -2714,20 +2704,18 @@ TEST_P(LdsRdsTest, XdsRoutingPrefixMatching) { {"locality0", GetBackendPorts(3, 4)}, }); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args1, kNewCluster1Name), - kNewCluster1Name); + AdsServiceImpl::BuildEdsResource(args1, kNewCluster1Name)); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args2, kNewCluster2Name), - kNewCluster2Name); + AdsServiceImpl::BuildEdsResource(args2, kNewCluster2Name)); // Populate new CDS resources. Cluster new_cluster1 = balancers_[0]->ads_service()->default_cluster(); new_cluster1.set_name(kNewCluster1Name); - balancers_[0]->ads_service()->SetCdsResource(new_cluster1, kNewCluster1Name); + balancers_[0]->ads_service()->SetCdsResource(new_cluster1); Cluster new_cluster2 = balancers_[0]->ads_service()->default_cluster(); new_cluster2.set_name(kNewCluster2Name); - balancers_[0]->ads_service()->SetCdsResource(new_cluster2, kNewCluster2Name); + balancers_[0]->ads_service()->SetCdsResource(new_cluster2); // Populating Route Configurations for LDS. RouteConfiguration new_route_config = balancers_[0]->ads_service()->default_route_config(); @@ -2780,7 +2768,7 @@ TEST_P(CdsTest, Vanilla) { TEST_P(CdsTest, WrongClusterType) { auto cluster = balancers_[0]->ads_service()->default_cluster(); cluster.set_type(envoy::api::v2::Cluster::STATIC); - balancers_[0]->ads_service()->SetCdsResource(cluster, kDefaultResourceName); + balancers_[0]->ads_service()->SetCdsResource(cluster); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); @@ -2795,7 +2783,7 @@ TEST_P(CdsTest, WrongClusterType) { TEST_P(CdsTest, WrongEdsConfig) { auto cluster = balancers_[0]->ads_service()->default_cluster(); cluster.mutable_eds_cluster_config()->mutable_eds_config()->mutable_self(); - balancers_[0]->ads_service()->SetCdsResource(cluster, kDefaultResourceName); + balancers_[0]->ads_service()->SetCdsResource(cluster); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); @@ -2810,7 +2798,7 @@ TEST_P(CdsTest, WrongEdsConfig) { TEST_P(CdsTest, WrongLbPolicy) { auto cluster = balancers_[0]->ads_service()->default_cluster(); cluster.set_lb_policy(envoy::api::v2::Cluster::LEAST_REQUEST); - balancers_[0]->ads_service()->SetCdsResource(cluster, kDefaultResourceName); + balancers_[0]->ads_service()->SetCdsResource(cluster); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); @@ -2825,7 +2813,7 @@ TEST_P(CdsTest, WrongLbPolicy) { TEST_P(CdsTest, WrongLrsServer) { auto cluster = balancers_[0]->ads_service()->default_cluster(); cluster.mutable_lrs_server()->mutable_ads(); - balancers_[0]->ads_service()->SetCdsResource(cluster, kDefaultResourceName); + balancers_[0]->ads_service()->SetCdsResource(cluster); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); CheckRpcSendFailure(); @@ -2863,7 +2851,7 @@ TEST_P(EdsTest, NacksSparsePriorityList) { {"locality0", GetBackendPorts(), kDefaultLocalityWeight, 1}, }); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); CheckRpcSendFailure(); const auto& response_state = balancers_[0]->ads_service()->eds_response_state(); @@ -2893,7 +2881,7 @@ TEST_P(LocalityMapTest, WeightedRoundRobin) { {"locality1", GetBackendPorts(1, 2), kLocalityWeight1}, }); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); // Wait for both backends to be ready. WaitForAllBackends(0, 2); // Send kNumRpcs RPCs. @@ -2927,7 +2915,7 @@ TEST_P(LocalityMapTest, LocalityContainingNoEndpoints) { {"locality1", {}}, }); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); // Wait for both backends to be ready. WaitForAllBackends(); // Send kNumRpcs RPCs. @@ -2949,7 +2937,7 @@ TEST_P(LocalityMapTest, NoLocalities) { SetNextResolutionForLbChannelAllBalancers(); // EDS response contains 2 localities, one with no endpoints. balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource({}), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource({})); Status status = SendRpc(); EXPECT_FALSE(status.ok()); EXPECT_EQ(status.error_code(), StatusCode::UNAVAILABLE); @@ -2971,14 +2959,14 @@ TEST_P(LocalityMapTest, StressTest) { args.locality_list.emplace_back(std::move(locality)); } balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); // The second ADS response contains 1 locality, which contains backend 1. args = AdsServiceImpl::EdsResourceArgs({ {"locality0", GetBackendPorts(1, 2)}, }); - std::thread delayed_resource_setter(std::bind( - &BasicTest::SetEdsResourceWithDelay, this, 0, - AdsServiceImpl::BuildEdsResource(args), 60 * 1000, kDefaultResourceName)); + std::thread delayed_resource_setter( + std::bind(&BasicTest::SetEdsResourceWithDelay, this, 0, + AdsServiceImpl::BuildEdsResource(args), 60 * 1000)); // Wait until backend 0 is ready, before which kNumLocalities localities are // received and handled by the xds policy. WaitForBackend(0, /*reset_counters=*/false); @@ -3019,7 +3007,7 @@ TEST_P(LocalityMapTest, UpdateMap) { {"locality2", GetBackendPorts(2, 3), 4}, }); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); // Wait for the first 3 backends to be ready. WaitForAllBackends(0, 3); gpr_log(GPR_INFO, "========= BEFORE FIRST BATCH =========="); @@ -3050,7 +3038,7 @@ TEST_P(LocalityMapTest, UpdateMap) { {"locality3", GetBackendPorts(3, 4), 6}, }); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); // Backend 3 hasn't received any request. EXPECT_EQ(0U, backends_[3]->backend_service()->request_count()); // Wait until the locality update has been processed, as signaled by backend 3 @@ -3090,13 +3078,13 @@ TEST_P(LocalityMapTest, ReplaceAllLocalitiesInPriority) { {"locality0", GetBackendPorts(0, 1)}, }); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); args = AdsServiceImpl::EdsResourceArgs({ {"locality1", GetBackendPorts(1, 2)}, }); - std::thread delayed_resource_setter(std::bind( - &BasicTest::SetEdsResourceWithDelay, this, 0, - AdsServiceImpl::BuildEdsResource(args), 5000, kDefaultResourceName)); + std::thread delayed_resource_setter( + std::bind(&BasicTest::SetEdsResourceWithDelay, this, 0, + AdsServiceImpl::BuildEdsResource(args), 5000)); // Wait for the first backend to be ready. WaitForBackend(0); // Keep sending RPCs until we switch over to backend 1, which tells us @@ -3125,7 +3113,7 @@ TEST_P(FailoverTest, ChooseHighestPriority) { {"locality3", GetBackendPorts(3, 4), kDefaultLocalityWeight, 0}, }); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); WaitForBackend(3, false); for (size_t i = 0; i < 3; ++i) { EXPECT_EQ(0U, backends_[i]->backend_service()->request_count()); @@ -3143,7 +3131,7 @@ TEST_P(FailoverTest, DoesNotUsePriorityWithNoEndpoints) { {"locality3", {}, kDefaultLocalityWeight, 0}, }); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); WaitForBackend(0, false); for (size_t i = 1; i < 3; ++i) { EXPECT_EQ(0U, backends_[i]->backend_service()->request_count()); @@ -3159,7 +3147,7 @@ TEST_P(FailoverTest, DoesNotUseLocalityWithNoEndpoints) { {"locality1", GetBackendPorts(), kDefaultLocalityWeight, 0}, }); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); // Wait for all backends to be used. std::tuple counts = WaitForAllBackends(); // Make sure no RPCs failed in the transition. @@ -3180,7 +3168,7 @@ TEST_P(FailoverTest, Failover) { ShutdownBackend(3); ShutdownBackend(0); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); WaitForBackend(1, false); for (size_t i = 0; i < 4; ++i) { if (i == 1) continue; @@ -3203,7 +3191,7 @@ TEST_P(FailoverTest, SwitchBackToHigherPriority) { ShutdownBackend(3); ShutdownBackend(0); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); WaitForBackend(1, false); for (size_t i = 0; i < 4; ++i) { if (i == 1) continue; @@ -3225,7 +3213,7 @@ TEST_P(FailoverTest, UpdateInitialUnavailable) { {"locality1", GetBackendPorts(1, 2), kDefaultLocalityWeight, 1}, }); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); args = AdsServiceImpl::EdsResourceArgs({ {"locality0", GetBackendPorts(0, 1), kDefaultLocalityWeight, 0}, {"locality1", GetBackendPorts(1, 2), kDefaultLocalityWeight, 1}, @@ -3234,9 +3222,9 @@ TEST_P(FailoverTest, UpdateInitialUnavailable) { }); ShutdownBackend(0); ShutdownBackend(1); - std::thread delayed_resource_setter(std::bind( - &BasicTest::SetEdsResourceWithDelay, this, 0, - AdsServiceImpl::BuildEdsResource(args), 1000, kDefaultResourceName)); + std::thread delayed_resource_setter( + std::bind(&BasicTest::SetEdsResourceWithDelay, this, 0, + AdsServiceImpl::BuildEdsResource(args), 1000)); gpr_timespec deadline = gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), gpr_time_from_millis(500, GPR_TIMESPAN)); // Send 0.5 second worth of RPCs. @@ -3264,16 +3252,16 @@ TEST_P(FailoverTest, UpdatePriority) { {"locality3", GetBackendPorts(3, 4), kDefaultLocalityWeight, 0}, }); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); args = AdsServiceImpl::EdsResourceArgs({ {"locality0", GetBackendPorts(0, 1), kDefaultLocalityWeight, 2}, {"locality1", GetBackendPorts(1, 2), kDefaultLocalityWeight, 0}, {"locality2", GetBackendPorts(2, 3), kDefaultLocalityWeight, 1}, {"locality3", GetBackendPorts(3, 4), kDefaultLocalityWeight, 3}, }); - std::thread delayed_resource_setter(std::bind( - &BasicTest::SetEdsResourceWithDelay, this, 0, - AdsServiceImpl::BuildEdsResource(args), 1000, kDefaultResourceName)); + std::thread delayed_resource_setter( + std::bind(&BasicTest::SetEdsResourceWithDelay, this, 0, + AdsServiceImpl::BuildEdsResource(args), 1000)); WaitForBackend(3, false); for (size_t i = 0; i < 3; ++i) { EXPECT_EQ(0U, backends_[i]->backend_service()->request_count()); @@ -3297,7 +3285,7 @@ TEST_P(FailoverTest, MoveAllLocalitiesInCurrentPriorityToHigherPriority) { {"locality1", GetBackendPorts(1, 3), kDefaultLocalityWeight, 1}, }); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); // Second update: // - Priority 0 contains both localities 0 and 1. // - Priority 1 is not present. @@ -3307,9 +3295,9 @@ TEST_P(FailoverTest, MoveAllLocalitiesInCurrentPriorityToHigherPriority) { {"locality0", GetBackendPorts(0, 1), kDefaultLocalityWeight, 0}, {"locality1", GetBackendPorts(1, 4), kDefaultLocalityWeight, 0}, }); - std::thread delayed_resource_setter(std::bind( - &BasicTest::SetEdsResourceWithDelay, this, 0, - AdsServiceImpl::BuildEdsResource(args), 1000, kDefaultResourceName)); + std::thread delayed_resource_setter( + std::bind(&BasicTest::SetEdsResourceWithDelay, this, 0, + AdsServiceImpl::BuildEdsResource(args), 1000)); // When we get the first update, all backends in priority 0 are down, // so we will create priority 1. Backends 1 and 2 should have traffic, // but backend 3 should not. @@ -3343,7 +3331,7 @@ TEST_P(DropTest, Vanilla) { args.drop_categories = {{kLbDropType, kDropPerMillionForLb}, {kThrottleDropType, kDropPerMillionForThrottle}}; balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); WaitForAllBackends(); // Send kNumRpcs RPCs and count the drops. size_t num_drops = 0; @@ -3383,7 +3371,7 @@ TEST_P(DropTest, DropPerHundred) { args.drop_categories = {{kLbDropType, kDropPerHundredForLb}}; args.drop_denominator = FractionalPercent::HUNDRED; balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); WaitForAllBackends(); // Send kNumRpcs RPCs and count the drops. size_t num_drops = 0; @@ -3422,7 +3410,7 @@ TEST_P(DropTest, DropPerTenThousand) { args.drop_categories = {{kLbDropType, kDropPerTenThousandForLb}}; args.drop_denominator = FractionalPercent::TEN_THOUSAND; balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); WaitForAllBackends(); // Send kNumRpcs RPCs and count the drops. size_t num_drops = 0; @@ -3464,7 +3452,7 @@ TEST_P(DropTest, Update) { }); args.drop_categories = {{kLbDropType, kDropPerMillionForLb}}; balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); WaitForAllBackends(); // Send kNumRpcs RPCs and count the drops. size_t num_drops = 0; @@ -3495,7 +3483,7 @@ TEST_P(DropTest, Update) { args.drop_categories = {{kLbDropType, kDropPerMillionForLb}, {kThrottleDropType, kDropPerMillionForThrottle}}; balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); // Wait until the drop rate increases to the middle of the two configs, which // implies that the update has been in effect. const double kDropRateThreshold = @@ -3553,7 +3541,7 @@ TEST_P(DropTest, DropAll) { args.drop_categories = {{kLbDropType, kDropPerMillionForLb}, {kThrottleDropType, kDropPerMillionForThrottle}}; balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); // Send kNumRpcs RPCs and all of them are dropped. for (size_t i = 0; i < kNumRpcs; ++i) { EchoResponse response; @@ -3577,12 +3565,12 @@ TEST_P(BalancerUpdateTest, UpdateBalancersButKeepUsingOriginalBalancer) { {"locality0", {backends_[0]->port()}}, }); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); args = AdsServiceImpl::EdsResourceArgs({ {"locality0", {backends_[1]->port()}}, }); balancers_[1]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); // Wait until the first backend is ready. WaitForBackend(0); // Send 10 requests. @@ -3632,12 +3620,12 @@ TEST_P(BalancerUpdateTest, Repeated) { {"locality0", {backends_[0]->port()}}, }); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); args = AdsServiceImpl::EdsResourceArgs({ {"locality0", {backends_[1]->port()}}, }); balancers_[1]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); // Wait until the first backend is ready. WaitForBackend(0); // Send 10 requests. @@ -3698,12 +3686,12 @@ TEST_P(BalancerUpdateTest, DeadUpdate) { {"locality0", {backends_[0]->port()}}, }); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); args = AdsServiceImpl::EdsResourceArgs({ {"locality0", {backends_[1]->port()}}, }); balancers_[1]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); // Start servers and send 10 RPCs per server. gpr_log(GPR_INFO, "========= BEFORE FIRST BATCH =========="); CheckRpcSendOk(10); @@ -3784,7 +3772,7 @@ TEST_P(ClientLoadReportingTest, Vanilla) { {"locality0", GetBackendPorts()}, }); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); // Wait until all backends are ready. int num_ok = 0; int num_failure = 0; @@ -3829,7 +3817,7 @@ TEST_P(ClientLoadReportingTest, HonorsClustersRequestedByLrsServer) { {"locality0", GetBackendPorts()}, }); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); // Wait until all backends are ready. int num_ok = 0; int num_failure = 0; @@ -3863,7 +3851,7 @@ TEST_P(ClientLoadReportingTest, BalancerRestart) { {"locality0", GetBackendPorts(0, kNumBackendsFirstPass)}, }); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); // Wait until all backends returned by the balancer are ready. int num_ok = 0; int num_failure = 0; @@ -3901,7 +3889,7 @@ TEST_P(ClientLoadReportingTest, BalancerRestart) { {"locality0", GetBackendPorts(kNumBackendsFirstPass)}, }); balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); // Wait for queries to start going to one of the new backends. // This tells us that we're now using the new serverlist. std::tie(num_ok, num_failure, num_drops) = @@ -3943,7 +3931,7 @@ TEST_P(ClientLoadReportingWithDropTest, Vanilla) { args.drop_categories = {{kLbDropType, kDropPerMillionForLb}, {kThrottleDropType, kDropPerMillionForThrottle}}; balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); + AdsServiceImpl::BuildEdsResource(args)); int num_ok = 0; int num_failure = 0; int num_drops = 0;