From b416019f72a1b56ba11e60021d69bea9b4d42484 Mon Sep 17 00:00:00 2001 From: SataQiu Date: Tue, 16 Apr 2019 11:45:42 +0800 Subject: [PATCH 01/13] fix spelling errors of include/* --- include/grpc/grpc_security.h | 2 +- include/grpc/slice.h | 2 +- include/grpcpp/impl/codegen/completion_queue.h | 10 +++++----- include/grpcpp/impl/codegen/server_interface.h | 2 +- include/grpcpp/impl/codegen/sync_stream.h | 6 +++--- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/include/grpc/grpc_security.h b/include/grpc/grpc_security.h index f1185d2b2a6..53189097b9b 100644 --- a/include/grpc/grpc_security.h +++ b/include/grpc/grpc_security.h @@ -264,7 +264,7 @@ GRPCAPI grpc_call_credentials* grpc_google_refresh_token_credentials_create( const char* json_refresh_token, void* reserved); /** Creates an Oauth2 Access Token credentials with an access token that was - aquired by an out of band mechanism. */ + acquired by an out of band mechanism. */ GRPCAPI grpc_call_credentials* grpc_access_token_credentials_create( const char* access_token, void* reserved); diff --git a/include/grpc/slice.h b/include/grpc/slice.h index ce482922afa..192a8cfeef4 100644 --- a/include/grpc/slice.h +++ b/include/grpc/slice.h @@ -147,7 +147,7 @@ GPRAPI int grpc_slice_buf_start_eq(grpc_slice a, const void* b, size_t blen); GPRAPI int grpc_slice_rchr(grpc_slice s, char c); GPRAPI int grpc_slice_chr(grpc_slice s, char c); -/** return the index of the first occurance of \a needle in \a haystack, or -1 +/** return the index of the first occurrence of \a needle in \a haystack, or -1 if it's not found */ GPRAPI int grpc_slice_slice(grpc_slice haystack, grpc_slice needle); diff --git a/include/grpcpp/impl/codegen/completion_queue.h b/include/grpcpp/impl/codegen/completion_queue.h index 73556ce9899..49c281b807d 100644 --- a/include/grpcpp/impl/codegen/completion_queue.h +++ b/include/grpcpp/impl/codegen/completion_queue.h @@ -183,8 +183,8 @@ class CompletionQueue : private GrpcLibraryCodegen { /// within the \a deadline). A \a tag points to an arbitrary location usually /// employed to uniquely identify an event. /// - /// \param tag [out] Upon sucess, updated to point to the event's tag. - /// \param ok [out] Upon sucess, true if a successful event, false otherwise + /// \param tag [out] Upon success, updated to point to the event's tag. + /// \param ok [out] Upon success, true if a successful event, false otherwise /// See documentation for CompletionQueue::Next for explanation of ok /// \param deadline [in] How long to block in wait for an event. /// @@ -203,8 +203,8 @@ class CompletionQueue : private GrpcLibraryCodegen { /// employed to uniquely identify an event. /// /// \param f [in] Function to execute before calling AsyncNext on this queue. - /// \param tag [out] Upon sucess, updated to point to the event's tag. - /// \param ok [out] Upon sucess, true if read a regular event, false + /// \param tag [out] Upon success, updated to point to the event's tag. + /// \param ok [out] Upon success, true if read a regular event, false /// otherwise. /// \param deadline [in] How long to block in wait for an event. /// @@ -362,7 +362,7 @@ class CompletionQueue : private GrpcLibraryCodegen { /// queue should not really shutdown until all avalanching operations have /// been finalized. Note that we maintain the requirement that an avalanche /// registration must take place before CQ shutdown (which must be maintained - /// elsehwere) + /// elsewhere) void InitialAvalanching() { gpr_atm_rel_store(&avalanches_in_flight_, static_cast(1)); } diff --git a/include/grpcpp/impl/codegen/server_interface.h b/include/grpcpp/impl/codegen/server_interface.h index a0b0a580979..328f4389628 100644 --- a/include/grpcpp/impl/codegen/server_interface.h +++ b/include/grpcpp/impl/codegen/server_interface.h @@ -149,7 +149,7 @@ class ServerInterface : public internal::CallHook { /// 192.168.1.1:31416, [::1]:27182, etc.). /// \params creds The credentials associated with the server. /// - /// \return bound port number on sucess, 0 on failure. + /// \return bound port number on success, 0 on failure. /// /// \warning It's an error to call this method on an already started server. virtual int AddListeningPort(const grpc::string& addr, diff --git a/include/grpcpp/impl/codegen/sync_stream.h b/include/grpcpp/impl/codegen/sync_stream.h index d9edad42153..0d3fdfcb8dc 100644 --- a/include/grpcpp/impl/codegen/sync_stream.h +++ b/include/grpcpp/impl/codegen/sync_stream.h @@ -180,7 +180,7 @@ class ClientReader final : public ClientReaderInterface { /// // Side effect: /// Once complete, the initial metadata read from - /// the server will be accessable through the \a ClientContext used to + /// the server will be accessible through the \a ClientContext used to /// construct this object. void WaitForInitialMetadata() override { GPR_CODEGEN_ASSERT(!context_->initial_metadata_received_); @@ -298,7 +298,7 @@ class ClientWriter : public ClientWriterInterface { /// // Side effect: /// Once complete, the initial metadata read from the server will be - /// accessable through the \a ClientContext used to construct this object. + /// accessible through the \a ClientContext used to construct this object. void WaitForInitialMetadata() { GPR_CODEGEN_ASSERT(!context_->initial_metadata_received_); @@ -449,7 +449,7 @@ class ClientReaderWriter final : public ClientReaderWriterInterface { /// with or after the \a Finish method. /// /// Once complete, the initial metadata read from the server will be - /// accessable through the \a ClientContext used to construct this object. + /// accessible through the \a ClientContext used to construct this object. void WaitForInitialMetadata() override { GPR_CODEGEN_ASSERT(!context_->initial_metadata_received_); From 7095f5acbef42667bd59baf3a3ae9748bee44286 Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Tue, 16 Apr 2019 11:27:31 -0700 Subject: [PATCH 02/13] Use requirements.bazel.txt to generate Python documents --- tools/distrib/python/docgen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/distrib/python/docgen.py b/tools/distrib/python/docgen.py index de47c00618d..67d0054656c 100755 --- a/tools/distrib/python/docgen.py +++ b/tools/distrib/python/docgen.py @@ -44,7 +44,7 @@ PROJECT_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, '..', '..', '..')) CONFIG = args.config SETUP_PATH = os.path.join(PROJECT_ROOT, 'setup.py') -REQUIREMENTS_PATH = os.path.join(PROJECT_ROOT, 'requirements.txt') +REQUIREMENTS_PATH = os.path.join(PROJECT_ROOT, 'requirements.bazel.txt') DOC_PATH = os.path.join(PROJECT_ROOT, 'doc/build') INCLUDE_PATH = os.path.join(PROJECT_ROOT, 'include') LIBRARY_PATH = os.path.join(PROJECT_ROOT, 'libs/{}'.format(CONFIG)) From 541cb0047057a4d4d915d556c77e9fc610599f8c Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Mon, 25 Mar 2019 11:11:47 -0700 Subject: [PATCH 03/13] Add wait-for-ready example * With unit test * With Bazel integration * With REAME.md explaination --- examples/python/wait_for_ready/BUILD.bazel | 32 +++++ examples/python/wait_for_ready/README.md | 32 +++++ .../test/_wait_for_ready_example_test.py | 31 +++++ .../wait_for_ready/wait_for_ready_example.py | 112 ++++++++++++++++++ 4 files changed, 207 insertions(+) create mode 100644 examples/python/wait_for_ready/BUILD.bazel create mode 100644 examples/python/wait_for_ready/README.md create mode 100644 examples/python/wait_for_ready/test/_wait_for_ready_example_test.py create mode 100644 examples/python/wait_for_ready/wait_for_ready_example.py diff --git a/examples/python/wait_for_ready/BUILD.bazel b/examples/python/wait_for_ready/BUILD.bazel new file mode 100644 index 00000000000..70daf83d334 --- /dev/null +++ b/examples/python/wait_for_ready/BUILD.bazel @@ -0,0 +1,32 @@ +# Copyright 2019 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@grpc_python_dependencies//:requirements.bzl", "requirement") + +py_library( + name = "wait_for_ready_example", + testonly = 1, + srcs = ["wait_for_ready_example.py"], + deps = [ + "//src/python/grpcio/grpc:grpcio", + "//examples:py_helloworld", + ], +) + +py_test( + name = "test/_wait_for_ready_example_test", + srcs = ["test/_wait_for_ready_example_test.py"], + deps = [":wait_for_ready_example",], + size = "small", +) diff --git a/examples/python/wait_for_ready/README.md b/examples/python/wait_for_ready/README.md new file mode 100644 index 00000000000..5c9d19688e1 --- /dev/null +++ b/examples/python/wait_for_ready/README.md @@ -0,0 +1,32 @@ +# gRPC Python Example for Wait-for-ready + +The default behavior of an RPC will fail instantly if the server is not ready yet. This example demonstrates how to change that behavior. + + +### Definition of 'wait-for-ready' semantics +> If an RPC is issued but the channel is in TRANSIENT_FAILURE or SHUTDOWN states, the RPC is unable to be transmitted promptly. By default, gRPC implementations SHOULD fail such RPCs immediately. This is known as "fail fast," but the usage of the term is historical. RPCs SHOULD NOT fail as a result of the channel being in other states (CONNECTING, READY, or IDLE). +> +> gRPC implementations MAY provide a per-RPC option to not fail RPCs as a result of the channel being in TRANSIENT_FAILURE state. Instead, the implementation queues the RPCs until the channel is READY. This is known as "wait for ready." The RPCs SHOULD still fail before READY if there are unrelated reasons, such as the channel is SHUTDOWN or the RPC's deadline is reached. +> +> From https://github.com/grpc/grpc/blob/master/doc/wait-for-ready.md + + +### Use cases for 'wait-for-ready' + +When developers spin up gRPC clients and servers at the same time, it is very like to fail first couple RPC calls due to unavailability of the server. If developers failed to prepare for this situation, the result can be catastrophic. But with 'wait-for-ready' semantics, developers can initialize the client and server in any order, especially useful in testing. + +Also, developers may ensure the server is up before starting client. But in some cases like transient network failure may result in a temporary unavailability of the server. With 'wait-for-ready' semantics, those RPC calls will automatically wait until the server is ready to accept incoming requests. + + +### DEMO Snippets + +```Python +# Per RPC level +stub = ...Stub(...) + +stub.important_transaction_1(..., wait_for_ready=True) +stub.unimportant_transaction_2(...) +stub.important_transaction_3(..., wait_for_ready=True) +stub.unimportant_transaction_4(...) +# The unimportant transactions can be status report, or health check, etc. +``` diff --git a/examples/python/wait_for_ready/test/_wait_for_ready_example_test.py b/examples/python/wait_for_ready/test/_wait_for_ready_example_test.py new file mode 100644 index 00000000000..03e83a12c56 --- /dev/null +++ b/examples/python/wait_for_ready/test/_wait_for_ready_example_test.py @@ -0,0 +1,31 @@ +# Copyright 2019 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests of the wait-for-ready example.""" + +import unittest +import logging + +from examples.python.wait_for_ready import wait_for_ready_example + + +class WaitForReadyExampleTest(unittest.TestCase): + + def test_wait_for_ready_example(self): + wait_for_ready_example.main() + # No unhandled exception raised, no deadlock, test passed! + + +if __name__ == '__main__': + logging.basicConfig() + unittest.main(verbosity=2) diff --git a/examples/python/wait_for_ready/wait_for_ready_example.py b/examples/python/wait_for_ready/wait_for_ready_example.py new file mode 100644 index 00000000000..91d5a7c50cd --- /dev/null +++ b/examples/python/wait_for_ready/wait_for_ready_example.py @@ -0,0 +1,112 @@ +# Copyright 2019 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The Python example of utilizing wait-for-ready flag.""" + +from __future__ import print_function +import logging +from concurrent import futures +import socket +import threading + +import grpc + +from examples.protos import helloworld_pb2 +from examples.protos import helloworld_pb2_grpc + +_LOGGER = logging.getLogger(__name__) +_LOGGER.setLevel(logging.INFO) + +_ONE_DAY_IN_SECONDS = 60 * 60 * 24 + + +def get_free_loopback_tcp_port(): + tcp = socket.socket(socket.AF_INET6) + tcp.bind(('', 0)) + address_tuple = tcp.getsockname() + return tcp, "[::1]:%s" % (address_tuple[1]) + + +class Greeter(helloworld_pb2_grpc.GreeterServicer): + + def SayHello(self, request, unused_context): + return helloworld_pb2.HelloReply(message='Hello, %s!' % request.name) + + +def create_server(server_address): + server = grpc.server(futures.ThreadPoolExecutor()) + helloworld_pb2_grpc.add_GreeterServicer_to_server(Greeter(), server) + bound_port = server.add_insecure_port(server_address) + assert bound_port == int(server_address.split(':')[-1]) + return server + + +def process(stub, wait_for_ready=None): + try: + response = stub.SayHello( + helloworld_pb2.HelloRequest(name='you'), + wait_for_ready=wait_for_ready) + message = response.message + except grpc.RpcError as rpc_error: + assert rpc_error.code() == grpc.StatusCode.UNAVAILABLE + assert not wait_for_ready + message = rpc_error + else: + assert wait_for_ready + _LOGGER.info("Wait-for-ready %s, client received: %s", "enabled" + if wait_for_ready else "disabled", message) + + +def main(): + # Pick a random free port + tcp, server_address = get_free_loopback_tcp_port() + + # Register connectivity event to notify main thread + transient_failure_event = threading.Event() + + def wait_for_transient_failure(channel_connectivity): + if channel_connectivity == grpc.ChannelConnectivity.TRANSIENT_FAILURE: + transient_failure_event.set() + + # Create gRPC channel + channel = grpc.insecure_channel(server_address) + channel.subscribe(wait_for_transient_failure) + stub = helloworld_pb2_grpc.GreeterStub(channel) + + # Fire an RPC without wait_for_ready + thread_disabled_wait_for_ready = threading.Thread( + target=process, args=(stub, False)) + thread_disabled_wait_for_ready.start() + # Fire an RPC with wait_for_ready + thread_enabled_wait_for_ready = threading.Thread( + target=process, args=(stub, True)) + thread_enabled_wait_for_ready.start() + + # Wait for the channel entering TRANSIENT FAILURE state. + tcp.close() + transient_failure_event.wait() + server = create_server(server_address) + server.start() + + # Expected to fail with StatusCode.UNAVAILABLE. + thread_disabled_wait_for_ready.join() + # Expected to success. + thread_enabled_wait_for_ready.join() + + server.stop(None) + channel.close() + + +if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) + main() From f062722c6187fa5ec4c28a564465dc13c164c0df Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Tue, 16 Apr 2019 13:44:46 -0700 Subject: [PATCH 04/13] Adopt reviewer's advice * Use context manager to manage tcp socket * Rename tcp socket * Fix grammer error --- examples/python/wait_for_ready/README.md | 2 +- .../wait_for_ready/wait_for_ready_example.py | 48 ++++++++++--------- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/examples/python/wait_for_ready/README.md b/examples/python/wait_for_ready/README.md index 5c9d19688e1..6e873d2222d 100644 --- a/examples/python/wait_for_ready/README.md +++ b/examples/python/wait_for_ready/README.md @@ -1,6 +1,6 @@ # gRPC Python Example for Wait-for-ready -The default behavior of an RPC will fail instantly if the server is not ready yet. This example demonstrates how to change that behavior. +The default behavior of an RPC is to fail instantly if the server is not ready yet. This example demonstrates how to change that behavior. ### Definition of 'wait-for-ready' semantics diff --git a/examples/python/wait_for_ready/wait_for_ready_example.py b/examples/python/wait_for_ready/wait_for_ready_example.py index 91d5a7c50cd..7c16b9bd4a1 100644 --- a/examples/python/wait_for_ready/wait_for_ready_example.py +++ b/examples/python/wait_for_ready/wait_for_ready_example.py @@ -16,6 +16,7 @@ from __future__ import print_function import logging from concurrent import futures +from contextlib import contextmanager import socket import threading @@ -30,11 +31,13 @@ _LOGGER.setLevel(logging.INFO) _ONE_DAY_IN_SECONDS = 60 * 60 * 24 +@contextmanager def get_free_loopback_tcp_port(): - tcp = socket.socket(socket.AF_INET6) - tcp.bind(('', 0)) - address_tuple = tcp.getsockname() - return tcp, "[::1]:%s" % (address_tuple[1]) + tcp_socket = socket.socket(socket.AF_INET6) + tcp_socket.bind(('', 0)) + address_tuple = tcp_socket.getsockname() + yield "[::1]:%s" % (address_tuple[1]) + tcp_socket.close() class Greeter(helloworld_pb2_grpc.GreeterServicer): @@ -69,31 +72,30 @@ def process(stub, wait_for_ready=None): def main(): # Pick a random free port - tcp, server_address = get_free_loopback_tcp_port() + with get_free_loopback_tcp_port() as server_address: - # Register connectivity event to notify main thread - transient_failure_event = threading.Event() + # Register connectivity event to notify main thread + transient_failure_event = threading.Event() - def wait_for_transient_failure(channel_connectivity): - if channel_connectivity == grpc.ChannelConnectivity.TRANSIENT_FAILURE: - transient_failure_event.set() + def wait_for_transient_failure(channel_connectivity): + if channel_connectivity == grpc.ChannelConnectivity.TRANSIENT_FAILURE: + transient_failure_event.set() - # Create gRPC channel - channel = grpc.insecure_channel(server_address) - channel.subscribe(wait_for_transient_failure) - stub = helloworld_pb2_grpc.GreeterStub(channel) + # Create gRPC channel + channel = grpc.insecure_channel(server_address) + channel.subscribe(wait_for_transient_failure) + stub = helloworld_pb2_grpc.GreeterStub(channel) - # Fire an RPC without wait_for_ready - thread_disabled_wait_for_ready = threading.Thread( - target=process, args=(stub, False)) - thread_disabled_wait_for_ready.start() - # Fire an RPC with wait_for_ready - thread_enabled_wait_for_ready = threading.Thread( - target=process, args=(stub, True)) - thread_enabled_wait_for_ready.start() + # Fire an RPC without wait_for_ready + thread_disabled_wait_for_ready = threading.Thread( + target=process, args=(stub, False)) + thread_disabled_wait_for_ready.start() + # Fire an RPC with wait_for_ready + thread_enabled_wait_for_ready = threading.Thread( + target=process, args=(stub, True)) + thread_enabled_wait_for_ready.start() # Wait for the channel entering TRANSIENT FAILURE state. - tcp.close() transient_failure_event.wait() server = create_server(server_address) server.start() From 123fd943f130779e2e200b1c3227e54bce0ef4b1 Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Tue, 16 Apr 2019 14:13:04 -0700 Subject: [PATCH 05/13] Revert "Revert "Merge pull request #18547 from lidizheng/fix-gevent"" This reverts commit a922bd7a03a35af0aaceb8f79e71f234e3fb418c. --- .../resolver/dns/c_ares/dns_resolver_ares.cc | 8 ++- src/core/lib/iomgr/iomgr_custom.cc | 3 + src/core/lib/iomgr/iomgr_custom.h | 2 + src/python/grpcio_tests/commands.py | 8 ++- src/python/grpcio_tests/tests/tests.json | 1 + .../grpcio_tests/tests/unit/BUILD.bazel | 1 + .../tests/unit/_dns_resolver_test.py | 63 +++++++++++++++++++ 7 files changed, 83 insertions(+), 3 deletions(-) create mode 100644 src/python/grpcio_tests/tests/unit/_dns_resolver_test.py diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc index 7b5eb311393..6994f63bee4 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc @@ -43,6 +43,7 @@ #include "src/core/lib/gprpp/manual_constructor.h" #include "src/core/lib/iomgr/combiner.h" #include "src/core/lib/iomgr/gethostname.h" +#include "src/core/lib/iomgr/iomgr_custom.h" #include "src/core/lib/iomgr/resolve_address.h" #include "src/core/lib/iomgr/timer.h" #include "src/core/lib/json/json.h" @@ -430,8 +431,11 @@ static grpc_address_resolver_vtable ares_resolver = { grpc_resolve_address_ares, blocking_resolve_address_ares}; static bool should_use_ares(const char* resolver_env) { - return resolver_env == nullptr || strlen(resolver_env) == 0 || - gpr_stricmp(resolver_env, "ares") == 0; + // TODO(lidiz): Remove the "g_custom_iomgr_enabled" flag once c-ares support + // custom IO managers (e.g. gevent). + return !g_custom_iomgr_enabled && + (resolver_env == nullptr || strlen(resolver_env) == 0 || + gpr_stricmp(resolver_env, "ares") == 0); } void grpc_resolver_dns_ares_init() { diff --git a/src/core/lib/iomgr/iomgr_custom.cc b/src/core/lib/iomgr/iomgr_custom.cc index 56363c35fd6..f5ac8a0670a 100644 --- a/src/core/lib/iomgr/iomgr_custom.cc +++ b/src/core/lib/iomgr/iomgr_custom.cc @@ -49,6 +49,8 @@ static bool iomgr_platform_add_closure_to_background_poller( return false; } +bool g_custom_iomgr_enabled = false; + static grpc_iomgr_platform_vtable vtable = { iomgr_platform_init, iomgr_platform_flush, @@ -61,6 +63,7 @@ void grpc_custom_iomgr_init(grpc_socket_vtable* socket, grpc_custom_resolver_vtable* resolver, grpc_custom_timer_vtable* timer, grpc_custom_poller_vtable* poller) { + g_custom_iomgr_enabled = true; grpc_custom_endpoint_init(socket); grpc_custom_timer_init(timer); grpc_custom_pollset_init(poller); diff --git a/src/core/lib/iomgr/iomgr_custom.h b/src/core/lib/iomgr/iomgr_custom.h index 57cc2f9b923..e6a88843e5c 100644 --- a/src/core/lib/iomgr/iomgr_custom.h +++ b/src/core/lib/iomgr/iomgr_custom.h @@ -39,6 +39,8 @@ extern gpr_thd_id g_init_thread; #define GRPC_CUSTOM_IOMGR_ASSERT_SAME_THREAD() #endif /* GRPC_CUSTOM_IOMGR_THREAD_CHECK */ +extern bool g_custom_iomgr_enabled; + void grpc_custom_iomgr_init(grpc_socket_vtable* socket, grpc_custom_resolver_vtable* resolver, grpc_custom_timer_vtable* timer, diff --git a/src/python/grpcio_tests/commands.py b/src/python/grpcio_tests/commands.py index e9b6333c891..dc0795d4a12 100644 --- a/src/python/grpcio_tests/commands.py +++ b/src/python/grpcio_tests/commands.py @@ -154,6 +154,9 @@ class TestGevent(setuptools.Command): # TODO(https://github.com/grpc/grpc/issues/15411) enable this test 'unit._cython._channel_test.ChannelTest.test_negative_deadline_connectivity' ) + BANNED_WINDOWS_TESTS = ( + # TODO(https://github.com/grpc/grpc/pull/15411) enable this test + 'unit._dns_resolver_test.DNSResolverTest.test_connect_loopback',) description = 'run tests with gevent. Assumes grpc/gevent are installed' user_options = [] @@ -179,7 +182,10 @@ class TestGevent(setuptools.Command): loader = tests.Loader() loader.loadTestsFromNames(['tests']) runner = tests.Runner() - runner.skip_tests(self.BANNED_TESTS) + if sys.platform == 'win32': + runner.skip_tests(self.BANNED_TESTS + self.BANNED_WINDOWS_TESTS) + else: + runner.skip_tests(self.BANNED_TESTS) result = gevent.spawn(runner.run, loader.suite) result.join() if not result.value.wasSuccessful(): diff --git a/src/python/grpcio_tests/tests/tests.json b/src/python/grpcio_tests/tests/tests.json index 7729ca01d53..cc08d56248a 100644 --- a/src/python/grpcio_tests/tests/tests.json +++ b/src/python/grpcio_tests/tests/tests.json @@ -46,6 +46,7 @@ "unit._cython.cygrpc_test.InsecureServerInsecureClient", "unit._cython.cygrpc_test.SecureServerSecureClient", "unit._cython.cygrpc_test.TypeSmokeTest", + "unit._dns_resolver_test.DNSResolverTest", "unit._empty_message_test.EmptyMessageTest", "unit._error_message_encoding_test.ErrorMessageEncodingTest", "unit._exit_test.ExitTest", diff --git a/src/python/grpcio_tests/tests/unit/BUILD.bazel b/src/python/grpcio_tests/tests/unit/BUILD.bazel index 04f91e63a18..a161794f8be 100644 --- a/src/python/grpcio_tests/tests/unit/BUILD.bazel +++ b/src/python/grpcio_tests/tests/unit/BUILD.bazel @@ -14,6 +14,7 @@ GRPCIO_TESTS_UNIT = [ "_channel_ready_future_test.py", "_compression_test.py", "_credentials_test.py", + "_dns_resolver_test.py", "_empty_message_test.py", "_exit_test.py", "_interceptor_test.py", diff --git a/src/python/grpcio_tests/tests/unit/_dns_resolver_test.py b/src/python/grpcio_tests/tests/unit/_dns_resolver_test.py new file mode 100644 index 00000000000..d119707b19d --- /dev/null +++ b/src/python/grpcio_tests/tests/unit/_dns_resolver_test.py @@ -0,0 +1,63 @@ +# Copyright 2019 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for an actual dns resolution.""" + +import unittest +import logging +import six + +import grpc +from tests.unit import test_common +from tests.unit.framework.common import test_constants + +_METHOD = '/ANY/METHOD' +_REQUEST = b'\x00\x00\x00' +_RESPONSE = _REQUEST + + +class GenericHandler(grpc.GenericRpcHandler): + + def service(self, unused_handler_details): + return grpc.unary_unary_rpc_method_handler( + lambda request, unused_context: request, + ) + + +class DNSResolverTest(unittest.TestCase): + + def setUp(self): + self._server = test_common.test_server() + self._server.add_generic_rpc_handlers((GenericHandler(),)) + self._port = self._server.add_insecure_port('[::]:0') + self._server.start() + + def tearDown(self): + self._server.stop(None) + + def test_connect_loopback(self): + # NOTE(https://github.com/grpc/grpc/issues/18422) + # In short, Gevent + C-Ares = Segfault. The C-Ares driver is not + # supported by custom io manager like "gevent" or "libuv". + with grpc.insecure_channel( + 'loopback4.unittest.grpc.io:%d' % self._port) as channel: + self.assertEqual( + channel.unary_unary(_METHOD)( + _REQUEST, + timeout=test_constants.SHORT_TIMEOUT, + ), _RESPONSE) + + +if __name__ == '__main__': + logging.basicConfig() + unittest.main(verbosity=2) From 09d18aa6592b14950042e22ab4354bc1de6d074c Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Tue, 16 Apr 2019 15:30:48 -0700 Subject: [PATCH 06/13] Propagate KeyboardInterrupt above completion queue --- src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pxd.pxi | 2 +- src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pxd.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pxd.pxi index 9f06ce086ee..0307f74cbef 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pxd.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pxd.pxi @@ -13,7 +13,7 @@ # limitations under the License. -cdef grpc_event _next(grpc_completion_queue *c_completion_queue, deadline) +cdef grpc_event _next(grpc_completion_queue *c_completion_queue, deadline) except * cdef _interpret_event(grpc_event c_event) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi index 212d27dc2b7..325e72afa0a 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi @@ -20,7 +20,7 @@ import time cdef int _INTERRUPT_CHECK_PERIOD_MS = 200 -cdef grpc_event _next(grpc_completion_queue *c_completion_queue, deadline): +cdef grpc_event _next(grpc_completion_queue *c_completion_queue, deadline) except *: cdef gpr_timespec c_increment cdef gpr_timespec c_timeout cdef gpr_timespec c_deadline From 2b9448a71c39e5ebcd79bedc71d1da8efcdbfceb Mon Sep 17 00:00:00 2001 From: Soheil Hassas Yeganeh Date: Tue, 16 Apr 2019 14:51:16 -0400 Subject: [PATCH 07/13] Revert "Revert "Introduce C++ wrappers for gpr_mu and gpr_cv."" This reverts commit d09c9f8e20363918d6b1aa92ee977e6f62551b48. --- BUILD | 14 +- BUILD.gn | 5 +- CMakeLists.txt | 5 + Makefile | 5 + build.yaml | 8 +- gRPC-C++.podspec | 7 +- gRPC-Core.podspec | 4 +- grpc.gemspec | 2 +- include/grpcpp/channel.h | 3 +- include/grpcpp/impl/codegen/client_context.h | 3 +- include/grpcpp/impl/codegen/sync.h | 138 ++++++++++++++++++ include/grpcpp/server.h | 8 +- include/grpcpp/server_impl.h | 8 +- package.xml | 2 +- .../filters/client_channel/client_channel.cc | 2 +- .../health/health_check_client.cc | 4 +- .../health/health_check_client.h | 3 +- .../client_channel/http_connect_handshaker.cc | 2 +- .../client_channel/lb_policy/grpclb/grpclb.cc | 1 - .../lb_policy/grpclb/grpclb_client_stats.cc | 2 +- .../lb_policy/grpclb/grpclb_client_stats.h | 6 +- .../lb_policy/pick_first/pick_first.cc | 6 +- .../lb_policy/round_robin/round_robin.cc | 6 +- .../client_channel/lb_policy/xds/xds.cc | 19 +-- .../client_channel/resolving_lb_policy.cc | 2 +- .../ext/filters/client_channel/subchannel.cc | 58 ++++---- .../ext/filters/client_channel/subchannel.h | 3 +- src/core/lib/channel/channelz_registry.cc | 2 +- src/core/lib/channel/handshaker.h | 2 +- src/core/lib/gprpp/mutex_lock.h | 42 ------ src/core/lib/gprpp/sync.h | 126 ++++++++++++++++ src/core/lib/iomgr/ev_epollex_linux.cc | 2 +- src/core/lib/surface/init.cc | 2 +- .../ssl/session_cache/ssl_session_cache.cc | 2 +- src/cpp/client/channel_cc.cc | 2 +- src/cpp/client/client_context.cc | 5 +- src/cpp/server/dynamic_thread_pool.cc | 23 +-- src/cpp/server/dynamic_thread_pool.h | 7 +- .../health/default_health_check_service.cc | 28 ++-- .../health/default_health_check_service.h | 7 +- src/cpp/server/load_reporter/load_reporter.cc | 18 +-- src/cpp/server/load_reporter/load_reporter.h | 5 +- .../load_reporter_async_service_impl.cc | 24 +-- .../load_reporter_async_service_impl.h | 3 +- src/cpp/server/server_cc.cc | 24 +-- src/cpp/server/server_context.cc | 17 +-- src/cpp/thread_manager/thread_manager.cc | 34 ++--- src/cpp/thread_manager/thread_manager.h | 7 +- test/cpp/client/client_channel_stress_test.cc | 17 ++- test/cpp/end2end/client_lb_end2end_test.cc | 37 ++--- test/cpp/end2end/grpclb_end2end_test.cc | 67 ++++----- test/cpp/end2end/thread_stress_test.cc | 21 +-- test/cpp/end2end/xds_end2end_test.cc | 66 ++++----- tools/doxygen/Doxyfile.c++ | 1 + tools/doxygen/Doxyfile.c++.internal | 3 +- tools/doxygen/Doxyfile.core.internal | 2 +- .../generated/sources_and_headers.json | 20 ++- 57 files changed, 606 insertions(+), 336 deletions(-) create mode 100644 include/grpcpp/impl/codegen/sync.h delete mode 100644 src/core/lib/gprpp/mutex_lock.h create mode 100644 src/core/lib/gprpp/sync.h diff --git a/BUILD b/BUILD index fd75012d214..56d332e0807 100644 --- a/BUILD +++ b/BUILD @@ -525,6 +525,17 @@ grpc_cc_library( ], ) +grpc_cc_library( + name = "grpc++_internal_hdrs_only", + hdrs = [ + "include/grpcpp/impl/codegen/sync.h", + ], + language = "c++", + deps = [ + "gpr_codegen", + ], +) + grpc_cc_library( name = "gpr_base", srcs = [ @@ -590,8 +601,8 @@ grpc_cc_library( "src/core/lib/gprpp/manual_constructor.h", "src/core/lib/gprpp/map.h", "src/core/lib/gprpp/memory.h", - "src/core/lib/gprpp/mutex_lock.h", "src/core/lib/gprpp/pair.h", + "src/core/lib/gprpp/sync.h", "src/core/lib/gprpp/thd.h", "src/core/lib/profiling/timers.h", ], @@ -2147,6 +2158,7 @@ grpc_cc_library( "include/grpcpp/impl/codegen/time.h", ], deps = [ + "grpc++_internal_hdrs_only", "grpc_codegen", ], ) diff --git a/BUILD.gn b/BUILD.gn index 21f567d9af4..10b514f8f2e 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -186,8 +186,8 @@ config("grpc_config") { "src/core/lib/gprpp/manual_constructor.h", "src/core/lib/gprpp/map.h", "src/core/lib/gprpp/memory.h", - "src/core/lib/gprpp/mutex_lock.h", "src/core/lib/gprpp/pair.h", + "src/core/lib/gprpp/sync.h", "src/core/lib/gprpp/thd.h", "src/core/lib/gprpp/thd_posix.cc", "src/core/lib/gprpp/thd_windows.cc", @@ -1066,6 +1066,7 @@ config("grpc_config") { "include/grpcpp/impl/codegen/status_code_enum.h", "include/grpcpp/impl/codegen/string_ref.h", "include/grpcpp/impl/codegen/stub_options.h", + "include/grpcpp/impl/codegen/sync.h", "include/grpcpp/impl/codegen/sync_stream.h", "include/grpcpp/impl/codegen/time.h", "include/grpcpp/impl/grpc_library.h", @@ -1161,12 +1162,12 @@ config("grpc_config") { "src/core/lib/gprpp/manual_constructor.h", "src/core/lib/gprpp/map.h", "src/core/lib/gprpp/memory.h", - "src/core/lib/gprpp/mutex_lock.h", "src/core/lib/gprpp/optional.h", "src/core/lib/gprpp/orphanable.h", "src/core/lib/gprpp/pair.h", "src/core/lib/gprpp/ref_counted.h", "src/core/lib/gprpp/ref_counted_ptr.h", + "src/core/lib/gprpp/sync.h", "src/core/lib/gprpp/thd.h", "src/core/lib/http/format_request.h", "src/core/lib/http/httpcli.h", diff --git a/CMakeLists.txt b/CMakeLists.txt index f2a4f698255..ee8712f1d38 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3187,6 +3187,7 @@ foreach(_hdr include/grpcpp/impl/codegen/stub_options.h include/grpcpp/impl/codegen/sync_stream.h include/grpcpp/impl/codegen/time.h + include/grpcpp/impl/codegen/sync.h include/grpc++/impl/codegen/proto_utils.h include/grpcpp/impl/codegen/proto_buffer_reader.h include/grpcpp/impl/codegen/proto_buffer_writer.h @@ -3790,6 +3791,7 @@ foreach(_hdr include/grpcpp/impl/codegen/stub_options.h include/grpcpp/impl/codegen/sync_stream.h include/grpcpp/impl/codegen/time.h + include/grpcpp/impl/codegen/sync.h include/grpc/census.h ) string(REPLACE "include/" "" _path ${_hdr}) @@ -4244,6 +4246,7 @@ foreach(_hdr include/grpc/impl/codegen/sync_generic.h include/grpc/impl/codegen/sync_posix.h include/grpc/impl/codegen/sync_windows.h + include/grpcpp/impl/codegen/sync.h include/grpc++/impl/codegen/proto_utils.h include/grpcpp/impl/codegen/proto_buffer_reader.h include/grpcpp/impl/codegen/proto_buffer_writer.h @@ -4440,6 +4443,7 @@ foreach(_hdr include/grpc/impl/codegen/sync_generic.h include/grpc/impl/codegen/sync_posix.h include/grpc/impl/codegen/sync_windows.h + include/grpcpp/impl/codegen/sync.h include/grpc++/impl/codegen/proto_utils.h include/grpcpp/impl/codegen/proto_buffer_reader.h include/grpcpp/impl/codegen/proto_buffer_writer.h @@ -4766,6 +4770,7 @@ foreach(_hdr include/grpcpp/impl/codegen/stub_options.h include/grpcpp/impl/codegen/sync_stream.h include/grpcpp/impl/codegen/time.h + include/grpcpp/impl/codegen/sync.h ) string(REPLACE "include/" "" _path ${_hdr}) get_filename_component(_path ${_path} PATH) diff --git a/Makefile b/Makefile index 6f19cc8d76b..c7fbca51225 100644 --- a/Makefile +++ b/Makefile @@ -5523,6 +5523,7 @@ PUBLIC_HEADERS_CXX += \ include/grpcpp/impl/codegen/stub_options.h \ include/grpcpp/impl/codegen/sync_stream.h \ include/grpcpp/impl/codegen/time.h \ + include/grpcpp/impl/codegen/sync.h \ include/grpc++/impl/codegen/proto_utils.h \ include/grpcpp/impl/codegen/proto_buffer_reader.h \ include/grpcpp/impl/codegen/proto_buffer_writer.h \ @@ -6134,6 +6135,7 @@ PUBLIC_HEADERS_CXX += \ include/grpcpp/impl/codegen/stub_options.h \ include/grpcpp/impl/codegen/sync_stream.h \ include/grpcpp/impl/codegen/time.h \ + include/grpcpp/impl/codegen/sync.h \ include/grpc/census.h \ LIBGRPC++_CRONET_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBGRPC++_CRONET_SRC)))) @@ -6560,6 +6562,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc/impl/codegen/sync_generic.h \ include/grpc/impl/codegen/sync_posix.h \ include/grpc/impl/codegen/sync_windows.h \ + include/grpcpp/impl/codegen/sync.h \ include/grpc++/impl/codegen/proto_utils.h \ include/grpcpp/impl/codegen/proto_buffer_reader.h \ include/grpcpp/impl/codegen/proto_buffer_writer.h \ @@ -6727,6 +6730,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc/impl/codegen/sync_generic.h \ include/grpc/impl/codegen/sync_posix.h \ include/grpc/impl/codegen/sync_windows.h \ + include/grpcpp/impl/codegen/sync.h \ include/grpc++/impl/codegen/proto_utils.h \ include/grpcpp/impl/codegen/proto_buffer_reader.h \ include/grpcpp/impl/codegen/proto_buffer_writer.h \ @@ -7059,6 +7063,7 @@ PUBLIC_HEADERS_CXX += \ include/grpcpp/impl/codegen/stub_options.h \ include/grpcpp/impl/codegen/sync_stream.h \ include/grpcpp/impl/codegen/time.h \ + include/grpcpp/impl/codegen/sync.h \ LIBGRPC++_UNSECURE_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBGRPC++_UNSECURE_SRC)))) diff --git a/build.yaml b/build.yaml index 6683db05fd4..4593be986bf 100644 --- a/build.yaml +++ b/build.yaml @@ -196,8 +196,8 @@ filegroups: - src/core/lib/gprpp/manual_constructor.h - src/core/lib/gprpp/map.h - src/core/lib/gprpp/memory.h - - src/core/lib/gprpp/mutex_lock.h - src/core/lib/gprpp/pair.h + - src/core/lib/gprpp/sync.h - src/core/lib/gprpp/thd.h - src/core/lib/profiling/timers.h uses: @@ -1278,6 +1278,7 @@ filegroups: - include/grpcpp/impl/codegen/time.h uses: - grpc_codegen + - grpc++_internal_hdrs_only - name: grpc++_codegen_base_src language: c++ src: @@ -1452,6 +1453,7 @@ filegroups: - grpc_base_headers - grpc_transport_inproc_headers - grpc++_codegen_base + - grpc++_internal_hdrs_only - nanopb_headers - health_proto - name: grpc++_config_proto @@ -1459,6 +1461,10 @@ filegroups: public_headers: - include/grpc++/impl/codegen/config_protobuf.h - include/grpcpp/impl/codegen/config_protobuf.h +- name: grpc++_internal_hdrs_only + language: c++ + public_headers: + - include/grpcpp/impl/codegen/sync.h - name: grpc++_reflection_proto language: c++ src: diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index 4d858ab6c77..74f758e6487 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -183,7 +183,8 @@ Pod::Spec.new do |s| 'include/grpcpp/impl/codegen/string_ref.h', 'include/grpcpp/impl/codegen/stub_options.h', 'include/grpcpp/impl/codegen/sync_stream.h', - 'include/grpcpp/impl/codegen/time.h' + 'include/grpcpp/impl/codegen/time.h', + 'include/grpcpp/impl/codegen/sync.h' end s.subspec 'Implementation' do |ss| @@ -266,8 +267,8 @@ Pod::Spec.new do |s| 'src/core/lib/gprpp/manual_constructor.h', 'src/core/lib/gprpp/map.h', 'src/core/lib/gprpp/memory.h', - 'src/core/lib/gprpp/mutex_lock.h', 'src/core/lib/gprpp/pair.h', + 'src/core/lib/gprpp/sync.h', 'src/core/lib/gprpp/thd.h', 'src/core/lib/profiling/timers.h', 'src/core/ext/transport/chttp2/transport/bin_decoder.h', @@ -584,8 +585,8 @@ Pod::Spec.new do |s| 'src/core/lib/gprpp/manual_constructor.h', 'src/core/lib/gprpp/map.h', 'src/core/lib/gprpp/memory.h', - 'src/core/lib/gprpp/mutex_lock.h', 'src/core/lib/gprpp/pair.h', + 'src/core/lib/gprpp/sync.h', 'src/core/lib/gprpp/thd.h', 'src/core/lib/profiling/timers.h', 'src/core/lib/avl/avl.h', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index b54ec902f92..5cf45c63cef 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -210,8 +210,8 @@ Pod::Spec.new do |s| 'src/core/lib/gprpp/manual_constructor.h', 'src/core/lib/gprpp/map.h', 'src/core/lib/gprpp/memory.h', - 'src/core/lib/gprpp/mutex_lock.h', 'src/core/lib/gprpp/pair.h', + 'src/core/lib/gprpp/sync.h', 'src/core/lib/gprpp/thd.h', 'src/core/lib/profiling/timers.h', 'src/core/lib/gpr/alloc.cc', @@ -891,8 +891,8 @@ Pod::Spec.new do |s| 'src/core/lib/gprpp/manual_constructor.h', 'src/core/lib/gprpp/map.h', 'src/core/lib/gprpp/memory.h', - 'src/core/lib/gprpp/mutex_lock.h', 'src/core/lib/gprpp/pair.h', + 'src/core/lib/gprpp/sync.h', 'src/core/lib/gprpp/thd.h', 'src/core/lib/profiling/timers.h', 'src/core/ext/transport/chttp2/transport/bin_decoder.h', diff --git a/grpc.gemspec b/grpc.gemspec index c7ceac4a9ee..15c26f11569 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -104,8 +104,8 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/gprpp/manual_constructor.h ) s.files += %w( src/core/lib/gprpp/map.h ) s.files += %w( src/core/lib/gprpp/memory.h ) - s.files += %w( src/core/lib/gprpp/mutex_lock.h ) s.files += %w( src/core/lib/gprpp/pair.h ) + s.files += %w( src/core/lib/gprpp/sync.h ) s.files += %w( src/core/lib/gprpp/thd.h ) s.files += %w( src/core/lib/profiling/timers.h ) s.files += %w( src/core/lib/gpr/alloc.cc ) diff --git a/include/grpcpp/channel.h b/include/grpcpp/channel.h index ee833960698..c4d5ab1177b 100644 --- a/include/grpcpp/channel.h +++ b/include/grpcpp/channel.h @@ -28,6 +28,7 @@ #include #include #include +#include struct grpc_channel; @@ -97,7 +98,7 @@ class Channel final : public ChannelInterface, grpc_channel* const c_channel_; // owned // mu_ protects callback_cq_ (the per-channel callbackable completion queue) - std::mutex mu_; + grpc::internal::Mutex mu_; // callback_cq_ references the callbackable completion queue associated // with this channel (if any). It is set on the first call to CallbackCQ(). diff --git a/include/grpcpp/impl/codegen/client_context.h b/include/grpcpp/impl/codegen/client_context.h index 5946488566e..85bbf36f06d 100644 --- a/include/grpcpp/impl/codegen/client_context.h +++ b/include/grpcpp/impl/codegen/client_context.h @@ -51,6 +51,7 @@ #include #include #include +#include #include struct census_context; @@ -457,7 +458,7 @@ class ClientContext { bool idempotent_; bool cacheable_; std::shared_ptr channel_; - std::mutex mu_; + grpc::internal::Mutex mu_; grpc_call* call_; bool call_canceled_; gpr_timespec deadline_; diff --git a/include/grpcpp/impl/codegen/sync.h b/include/grpcpp/impl/codegen/sync.h new file mode 100644 index 00000000000..2ed71eeb9f2 --- /dev/null +++ b/include/grpcpp/impl/codegen/sync.h @@ -0,0 +1,138 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPCPP_IMPL_CODEGEN_SYNC_H +#define GRPCPP_IMPL_CODEGEN_SYNC_H + +#include +#include +#include + +#include + +// The core library is not accessible in C++ codegen headers, and vice versa. +// Thus, we need to have duplicate headers with similar functionality. +// Make sure any change to this file is also reflected in +// src/core/lib/gprpp/sync.h too. +// +// Whenever possible, prefer "src/core/lib/gprpp/sync.h" over this file, +// since in core we do not rely on g_core_codegen_interface and hence do not +// pay the costs of virtual function calls. + +namespace grpc { +namespace internal { + +class Mutex { + public: + Mutex() { g_core_codegen_interface->gpr_mu_init(&mu_); } + ~Mutex() { g_core_codegen_interface->gpr_mu_destroy(&mu_); } + + Mutex(const Mutex&) = delete; + Mutex& operator=(const Mutex&) = delete; + + gpr_mu* get() { return &mu_; } + const gpr_mu* get() const { return &mu_; } + + private: + gpr_mu mu_; +}; + +// MutexLock is a std:: +class MutexLock { + public: + explicit MutexLock(Mutex* mu) : mu_(mu->get()) { + g_core_codegen_interface->gpr_mu_lock(mu_); + } + explicit MutexLock(gpr_mu* mu) : mu_(mu) { + g_core_codegen_interface->gpr_mu_lock(mu_); + } + ~MutexLock() { g_core_codegen_interface->gpr_mu_unlock(mu_); } + + MutexLock(const MutexLock&) = delete; + MutexLock& operator=(const MutexLock&) = delete; + + private: + gpr_mu* const mu_; +}; + +class ReleasableMutexLock { + public: + explicit ReleasableMutexLock(Mutex* mu) : mu_(mu->get()) { + g_core_codegen_interface->gpr_mu_lock(mu_); + } + explicit ReleasableMutexLock(gpr_mu* mu) : mu_(mu) { + g_core_codegen_interface->gpr_mu_lock(mu_); + } + ~ReleasableMutexLock() { + if (!released_) g_core_codegen_interface->gpr_mu_unlock(mu_); + } + + ReleasableMutexLock(const ReleasableMutexLock&) = delete; + ReleasableMutexLock& operator=(const ReleasableMutexLock&) = delete; + + void Lock() { + GPR_DEBUG_ASSERT(released_); + g_core_codegen_interface->gpr_mu_lock(mu_); + released_ = false; + } + + void Unlock() { + GPR_DEBUG_ASSERT(!released_); + released_ = true; + g_core_codegen_interface->gpr_mu_unlock(mu_); + } + + private: + gpr_mu* const mu_; + bool released_ = false; +}; + +class CondVar { + public: + CondVar() { g_core_codegen_interface->gpr_cv_init(&cv_); } + ~CondVar() { g_core_codegen_interface->gpr_cv_destroy(&cv_); } + + CondVar(const CondVar&) = delete; + CondVar& operator=(const CondVar&) = delete; + + void Signal() { g_core_codegen_interface->gpr_cv_signal(&cv_); } + void Broadcast() { g_core_codegen_interface->gpr_cv_broadcast(&cv_); } + + int Wait(Mutex* mu) { + return Wait(mu, + g_core_codegen_interface->gpr_inf_future(GPR_CLOCK_REALTIME)); + } + int Wait(Mutex* mu, const gpr_timespec& deadline) { + return g_core_codegen_interface->gpr_cv_wait(&cv_, mu->get(), deadline); + } + + template + void WaitUntil(Mutex* mu, Predicate pred) { + while (!pred()) { + Wait(mu, g_core_codegen_interface->gpr_inf_future(GPR_CLOCK_REALTIME)); + } + } + + private: + gpr_cv cv_; +}; + +} // namespace internal +} // namespace grpc + +#endif // GRPCPP_IMPL_CODEGEN_SYNC_H diff --git a/include/grpcpp/server.h b/include/grpcpp/server.h index 2ae9d712012..8aff0663fe2 100644 --- a/include/grpcpp/server.h +++ b/include/grpcpp/server.h @@ -297,12 +297,12 @@ class Server : public ServerInterface, private GrpcLibraryCodegen { experimental_registration_type experimental_registration_{this}; // Server status - std::mutex mu_; + grpc::internal::Mutex mu_; bool started_; bool shutdown_; bool shutdown_notified_; // Was notify called on the shutdown_cv_ - std::condition_variable shutdown_cv_; + grpc::internal::CondVar shutdown_cv_; // It is ok (but not required) to nest callback_reqs_mu_ under mu_ . // Incrementing callback_reqs_outstanding_ is ok without a lock but it must be @@ -311,8 +311,8 @@ class Server : public ServerInterface, private GrpcLibraryCodegen { // during periods of increasing load; the decrement happens only when memory // is maxed out, during server shutdown, or (possibly in a future version) // during decreasing load, so it is less performance-critical. - std::mutex callback_reqs_mu_; - std::condition_variable callback_reqs_done_cv_; + grpc::internal::Mutex callback_reqs_mu_; + grpc::internal::CondVar callback_reqs_done_cv_; std::atomic_int callback_reqs_outstanding_{0}; std::shared_ptr global_callbacks_; diff --git a/include/grpcpp/server_impl.h b/include/grpcpp/server_impl.h index 771a3a10be9..14b16a06f4e 100644 --- a/include/grpcpp/server_impl.h +++ b/include/grpcpp/server_impl.h @@ -304,12 +304,12 @@ class Server : public grpc::ServerInterface, private grpc::GrpcLibraryCodegen { experimental_registration_type experimental_registration_{this}; // Server status - std::mutex mu_; + grpc::internal::Mutex mu_; bool started_; bool shutdown_; bool shutdown_notified_; // Was notify called on the shutdown_cv_ - std::condition_variable shutdown_cv_; + grpc::internal::CondVar shutdown_cv_; // It is ok (but not required) to nest callback_reqs_mu_ under mu_ . // Incrementing callback_reqs_outstanding_ is ok without a lock but it must be @@ -318,8 +318,8 @@ class Server : public grpc::ServerInterface, private grpc::GrpcLibraryCodegen { // during periods of increasing load; the decrement happens only when memory // is maxed out, during server shutdown, or (possibly in a future version) // during decreasing load, so it is less performance-critical. - std::mutex callback_reqs_mu_; - std::condition_variable callback_reqs_done_cv_; + grpc::internal::Mutex callback_reqs_mu_; + grpc::internal::CondVar callback_reqs_done_cv_; std::atomic_int callback_reqs_outstanding_{0}; std::shared_ptr global_callbacks_; diff --git a/package.xml b/package.xml index 1f08eaf4a20..c3f4e17dd12 100644 --- a/package.xml +++ b/package.xml @@ -109,8 +109,8 @@ - + diff --git a/src/core/ext/filters/client_channel/client_channel.cc b/src/core/ext/filters/client_channel/client_channel.cc index 0304a265a09..2ce7c99f16f 100644 --- a/src/core/ext/filters/client_channel/client_channel.cc +++ b/src/core/ext/filters/client_channel/client_channel.cc @@ -51,7 +51,7 @@ #include "src/core/lib/gpr/string.h" #include "src/core/lib/gprpp/inlined_vector.h" #include "src/core/lib/gprpp/manual_constructor.h" -#include "src/core/lib/gprpp/mutex_lock.h" +#include "src/core/lib/gprpp/sync.h" #include "src/core/lib/iomgr/combiner.h" #include "src/core/lib/iomgr/iomgr.h" #include "src/core/lib/iomgr/polling_entity.h" diff --git a/src/core/ext/filters/client_channel/health/health_check_client.cc b/src/core/ext/filters/client_channel/health/health_check_client.cc index a22d6450cbd..a99f1e54062 100644 --- a/src/core/ext/filters/client_channel/health/health_check_client.cc +++ b/src/core/ext/filters/client_channel/health/health_check_client.cc @@ -27,7 +27,7 @@ #include "pb_encode.h" #include "src/core/ext/filters/client_channel/health/health.pb.h" #include "src/core/lib/debug/trace.h" -#include "src/core/lib/gprpp/mutex_lock.h" +#include "src/core/lib/gprpp/sync.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/transport/error_utils.h" #include "src/core/lib/transport/status_metadata.h" @@ -69,7 +69,6 @@ HealthCheckClient::HealthCheckClient( } GRPC_CLOSURE_INIT(&retry_timer_callback_, OnRetryTimer, this, grpc_schedule_on_exec_ctx); - gpr_mu_init(&mu_); StartCall(); } @@ -78,7 +77,6 @@ HealthCheckClient::~HealthCheckClient() { gpr_log(GPR_INFO, "destroying HealthCheckClient %p", this); } GRPC_ERROR_UNREF(error_); - gpr_mu_destroy(&mu_); } void HealthCheckClient::NotifyOnHealthChange(grpc_connectivity_state* state, diff --git a/src/core/ext/filters/client_channel/health/health_check_client.h b/src/core/ext/filters/client_channel/health/health_check_client.h index 1fa4487c403..6e0123e4925 100644 --- a/src/core/ext/filters/client_channel/health/health_check_client.h +++ b/src/core/ext/filters/client_channel/health/health_check_client.h @@ -31,6 +31,7 @@ #include "src/core/lib/gprpp/atomic.h" #include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" +#include "src/core/lib/gprpp/sync.h" #include "src/core/lib/iomgr/call_combiner.h" #include "src/core/lib/iomgr/closure.h" #include "src/core/lib/iomgr/polling_entity.h" @@ -157,7 +158,7 @@ class HealthCheckClient : public InternallyRefCounted { grpc_pollset_set* interested_parties_; // Do not own. RefCountedPtr channelz_node_; - gpr_mu mu_; + Mutex mu_; grpc_connectivity_state state_ = GRPC_CHANNEL_CONNECTING; grpc_error* error_ = GRPC_ERROR_NONE; grpc_connectivity_state* notify_state_ = nullptr; diff --git a/src/core/ext/filters/client_channel/http_connect_handshaker.cc b/src/core/ext/filters/client_channel/http_connect_handshaker.cc index 2b1eb92bbd4..90a79843458 100644 --- a/src/core/ext/filters/client_channel/http_connect_handshaker.cc +++ b/src/core/ext/filters/client_channel/http_connect_handshaker.cc @@ -33,7 +33,7 @@ #include "src/core/lib/channel/handshaker_registry.h" #include "src/core/lib/gpr/env.h" #include "src/core/lib/gpr/string.h" -#include "src/core/lib/gprpp/mutex_lock.h" +#include "src/core/lib/gprpp/sync.h" #include "src/core/lib/http/format_request.h" #include "src/core/lib/http/parser.h" #include "src/core/lib/slice/slice_internal.h" diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc index 867a5c667fc..4423e479d62 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc @@ -88,7 +88,6 @@ #include "src/core/lib/gpr/string.h" #include "src/core/lib/gprpp/manual_constructor.h" #include "src/core/lib/gprpp/memory.h" -#include "src/core/lib/gprpp/mutex_lock.h" #include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/iomgr/combiner.h" diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc index 84b9c41a734..35123633feb 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc @@ -25,7 +25,7 @@ #include #include -#include "src/core/lib/gprpp/mutex_lock.h" +#include "src/core/lib/gprpp/sync.h" namespace grpc_core { diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h index fdebdf55c17..bcc6598c105 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h @@ -26,6 +26,7 @@ #include "src/core/lib/gprpp/inlined_vector.h" #include "src/core/lib/gprpp/memory.h" #include "src/core/lib/gprpp/ref_counted.h" +#include "src/core/lib/gprpp/sync.h" namespace grpc_core { @@ -41,9 +42,6 @@ class GrpcLbClientStats : public RefCounted { typedef InlinedVector DroppedCallCounts; - GrpcLbClientStats() { gpr_mu_init(&drop_count_mu_); } - ~GrpcLbClientStats() { gpr_mu_destroy(&drop_count_mu_); } - void AddCallStarted(); void AddCallFinished(bool finished_with_client_failed_to_send, bool finished_known_received); @@ -66,7 +64,7 @@ class GrpcLbClientStats : public RefCounted { gpr_atm num_calls_finished_ = 0; gpr_atm num_calls_finished_with_client_failed_to_send_ = 0; gpr_atm num_calls_finished_known_received_ = 0; - gpr_mu drop_count_mu_; // Guards drop_token_counts_. + Mutex drop_count_mu_; // Guards drop_token_counts_. UniquePtr drop_token_counts_; }; diff --git a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc index 35ca68f717b..637806b4804 100644 --- a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc +++ b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc @@ -27,7 +27,7 @@ #include "src/core/ext/filters/client_channel/server_address.h" #include "src/core/ext/filters/client_channel/subchannel.h" #include "src/core/lib/channel/channel_args.h" -#include "src/core/lib/gprpp/mutex_lock.h" +#include "src/core/lib/gprpp/sync.h" #include "src/core/lib/iomgr/combiner.h" #include "src/core/lib/iomgr/sockaddr_utils.h" #include "src/core/lib/transport/connectivity_state.h" @@ -154,13 +154,12 @@ class PickFirst : public LoadBalancingPolicy { /// Lock and data used to capture snapshots of this channels child /// channels and subchannels. This data is consumed by channelz. - gpr_mu child_refs_mu_; + Mutex child_refs_mu_; channelz::ChildRefsList child_subchannels_; channelz::ChildRefsList child_channels_; }; PickFirst::PickFirst(Args args) : LoadBalancingPolicy(std::move(args)) { - gpr_mu_init(&child_refs_mu_); if (grpc_lb_pick_first_trace.enabled()) { gpr_log(GPR_INFO, "Pick First %p created.", this); } @@ -170,7 +169,6 @@ PickFirst::~PickFirst() { if (grpc_lb_pick_first_trace.enabled()) { gpr_log(GPR_INFO, "Destroying Pick First %p", this); } - gpr_mu_destroy(&child_refs_mu_); GPR_ASSERT(subchannel_list_ == nullptr); GPR_ASSERT(latest_pending_subchannel_list_ == nullptr); } diff --git a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc index 74ad1893c6a..b913333fb45 100644 --- a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc +++ b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc @@ -36,8 +36,8 @@ #include "src/core/ext/filters/client_channel/subchannel.h" #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/debug/trace.h" -#include "src/core/lib/gprpp/mutex_lock.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" +#include "src/core/lib/gprpp/sync.h" #include "src/core/lib/iomgr/combiner.h" #include "src/core/lib/iomgr/sockaddr_utils.h" #include "src/core/lib/transport/connectivity_state.h" @@ -188,7 +188,7 @@ class RoundRobin : public LoadBalancingPolicy { bool shutdown_ = false; /// Lock and data used to capture snapshots of this channel's child /// channels and subchannels. This data is consumed by channelz. - gpr_mu child_refs_mu_; + Mutex child_refs_mu_; channelz::ChildRefsList child_subchannels_; channelz::ChildRefsList child_channels_; }; @@ -240,7 +240,6 @@ RoundRobin::PickResult RoundRobin::Picker::Pick(PickArgs* pick, // RoundRobin::RoundRobin(Args args) : LoadBalancingPolicy(std::move(args)) { - gpr_mu_init(&child_refs_mu_); if (grpc_lb_round_robin_trace.enabled()) { gpr_log(GPR_INFO, "[RR %p] Created", this); } @@ -250,7 +249,6 @@ RoundRobin::~RoundRobin() { if (grpc_lb_round_robin_trace.enabled()) { gpr_log(GPR_INFO, "[RR %p] Destroying Round Robin policy", this); } - gpr_mu_destroy(&child_refs_mu_); GPR_ASSERT(subchannel_list_ == nullptr); GPR_ASSERT(latest_pending_subchannel_list_ == nullptr); } diff --git a/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc b/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc index 225bae7aa1c..dd782ac53c3 100644 --- a/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc +++ b/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc @@ -89,9 +89,9 @@ #include "src/core/lib/gprpp/manual_constructor.h" #include "src/core/lib/gprpp/map.h" #include "src/core/lib/gprpp/memory.h" -#include "src/core/lib/gprpp/mutex_lock.h" #include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" +#include "src/core/lib/gprpp/sync.h" #include "src/core/lib/iomgr/combiner.h" #include "src/core/lib/iomgr/sockaddr.h" #include "src/core/lib/iomgr/sockaddr_utils.h" @@ -278,10 +278,8 @@ class XdsLb : public LoadBalancingPolicy { class LocalityEntry : public InternallyRefCounted { public: explicit LocalityEntry(RefCountedPtr parent) - : parent_(std::move(parent)) { - gpr_mu_init(&child_policy_mu_); - } - ~LocalityEntry() { gpr_mu_destroy(&child_policy_mu_); } + : parent_(std::move(parent)) {} + ~LocalityEntry() = default; void UpdateLocked(xds_grpclb_serverlist* serverlist, LoadBalancingPolicy::Config* child_policy_config, @@ -323,13 +321,10 @@ class XdsLb : public LoadBalancingPolicy { OrphanablePtr pending_child_policy_; // Lock held when modifying the value of child_policy_ or // pending_child_policy_. - gpr_mu child_policy_mu_; + Mutex child_policy_mu_; RefCountedPtr parent_; }; - LocalityMap() { gpr_mu_init(&child_refs_mu_); } - ~LocalityMap() { gpr_mu_destroy(&child_refs_mu_); } - void UpdateLocked(const LocalityList& locality_list, LoadBalancingPolicy::Config* child_policy_config, const grpc_channel_args* args, XdsLb* parent); @@ -343,7 +338,7 @@ class XdsLb : public LoadBalancingPolicy { Map, OrphanablePtr, StringLess> map_; // Lock held while filling child refs for all localities // inside the map - gpr_mu child_refs_mu_; + Mutex child_refs_mu_; }; struct LocalityServerlistEntry { @@ -397,7 +392,7 @@ class XdsLb : public LoadBalancingPolicy { // Mutex to protect the channel to the LB server. This is used when // processing a channelz request. // TODO(juanlishen): Replace this with atomic. - gpr_mu lb_chand_mu_; + Mutex lb_chand_mu_; // Timeout in milliseconds for the LB call. 0 means no deadline. int lb_call_timeout_ms_ = 0; @@ -1090,7 +1085,6 @@ XdsLb::XdsLb(Args args) : LoadBalancingPolicy(std::move(args)), locality_map_(), locality_serverlist_() { - gpr_mu_init(&lb_chand_mu_); // Record server name. const grpc_arg* arg = grpc_channel_args_find(args.args, GRPC_ARG_SERVER_URI); const char* server_uri = grpc_channel_arg_get_string(arg); @@ -1114,7 +1108,6 @@ XdsLb::XdsLb(Args args) } XdsLb::~XdsLb() { - gpr_mu_destroy(&lb_chand_mu_); gpr_free((void*)server_name_); grpc_channel_args_destroy(args_); locality_serverlist_.clear(); diff --git a/src/core/ext/filters/client_channel/resolving_lb_policy.cc b/src/core/ext/filters/client_channel/resolving_lb_policy.cc index 8f53c33b7fd..4ccd8be29c0 100644 --- a/src/core/ext/filters/client_channel/resolving_lb_policy.cc +++ b/src/core/ext/filters/client_channel/resolving_lb_policy.cc @@ -48,7 +48,7 @@ #include "src/core/lib/gpr/string.h" #include "src/core/lib/gprpp/inlined_vector.h" #include "src/core/lib/gprpp/manual_constructor.h" -#include "src/core/lib/gprpp/mutex_lock.h" +#include "src/core/lib/gprpp/sync.h" #include "src/core/lib/iomgr/combiner.h" #include "src/core/lib/iomgr/iomgr.h" #include "src/core/lib/iomgr/polling_entity.h" diff --git a/src/core/ext/filters/client_channel/subchannel.cc b/src/core/ext/filters/client_channel/subchannel.cc index 9fbec8c3859..e29cd0a6dc3 100644 --- a/src/core/ext/filters/client_channel/subchannel.cc +++ b/src/core/ext/filters/client_channel/subchannel.cc @@ -42,8 +42,8 @@ #include "src/core/lib/gpr/alloc.h" #include "src/core/lib/gprpp/debug_location.h" #include "src/core/lib/gprpp/manual_constructor.h" -#include "src/core/lib/gprpp/mutex_lock.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" +#include "src/core/lib/gprpp/sync.h" #include "src/core/lib/iomgr/sockaddr_utils.h" #include "src/core/lib/profiling/timers.h" #include "src/core/lib/slice/slice_internal.h" @@ -454,13 +454,14 @@ struct Subchannel::ExternalStateWatcher { grpc_pollset_set_del_pollset_set(w->subchannel->pollset_set_, w->pollset_set); } - gpr_mu_lock(&w->subchannel->mu_); - if (w->subchannel->external_state_watcher_list_ == w) { - w->subchannel->external_state_watcher_list_ = w->next; + { + MutexLock lock(&w->subchannel->mu_); + if (w->subchannel->external_state_watcher_list_ == w) { + w->subchannel->external_state_watcher_list_ = w->next; + } + if (w->next != nullptr) w->next->prev = w->prev; + if (w->prev != nullptr) w->prev->next = w->next; } - if (w->next != nullptr) w->next->prev = w->prev; - if (w->prev != nullptr) w->prev->next = w->next; - gpr_mu_unlock(&w->subchannel->mu_); GRPC_SUBCHANNEL_WEAK_UNREF(w->subchannel, "external_state_watcher+done"); Delete(w); GRPC_CLOSURE_SCHED(follow_up, GRPC_ERROR_REF(error)); @@ -582,7 +583,6 @@ Subchannel::Subchannel(SubchannelKey* key, grpc_connector* connector, "subchannel"); grpc_connectivity_state_init(&state_and_health_tracker_, GRPC_CHANNEL_IDLE, "subchannel"); - gpr_mu_init(&mu_); // Check whether we should enable health checking. const char* service_config_json = grpc_channel_arg_get_string( grpc_channel_args_find(args_, GRPC_ARG_SERVICE_CONFIG)); @@ -629,7 +629,6 @@ Subchannel::~Subchannel() { grpc_connector_unref(connector_); grpc_pollset_set_destroy(pollset_set_); Delete(key_); - gpr_mu_destroy(&mu_); } Subchannel* Subchannel::Create(grpc_connector* connector, @@ -903,7 +902,9 @@ void Subchannel::MaybeStartConnectingLocked() { void Subchannel::OnRetryAlarm(void* arg, grpc_error* error) { Subchannel* c = static_cast(arg); - gpr_mu_lock(&c->mu_); + // TODO(soheilhy): Once subchannel refcounting is simplified, we can get use + // MutexLock instead of ReleasableMutexLock, here. + ReleasableMutexLock lock(&c->mu_); c->have_retry_alarm_ = false; if (c->disconnected_) { error = GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING("Disconnected", @@ -917,9 +918,9 @@ void Subchannel::OnRetryAlarm(void* arg, grpc_error* error) { if (error == GRPC_ERROR_NONE) { gpr_log(GPR_INFO, "Failed to connect to channel, retrying"); c->ContinueConnectingLocked(); - gpr_mu_unlock(&c->mu_); + lock.Unlock(); } else { - gpr_mu_unlock(&c->mu_); + lock.Unlock(); GRPC_SUBCHANNEL_WEAK_UNREF(c, "connecting"); } GRPC_ERROR_UNREF(error); @@ -944,24 +945,25 @@ void Subchannel::OnConnectingFinished(void* arg, grpc_error* error) { auto* c = static_cast(arg); grpc_channel_args* delete_channel_args = c->connecting_result_.channel_args; GRPC_SUBCHANNEL_WEAK_REF(c, "on_connecting_finished"); - gpr_mu_lock(&c->mu_); - c->connecting_ = false; - if (c->connecting_result_.transport != nullptr && - c->PublishTransportLocked()) { - // Do nothing, transport was published. - } else if (c->disconnected_) { - GRPC_SUBCHANNEL_WEAK_UNREF(c, "connecting"); - } else { - gpr_log(GPR_INFO, "Connect failed: %s", grpc_error_string(error)); - c->SetConnectivityStateLocked(GRPC_CHANNEL_TRANSIENT_FAILURE, + { + MutexLock lock(&c->mu_); + c->connecting_ = false; + if (c->connecting_result_.transport != nullptr && + c->PublishTransportLocked()) { + // Do nothing, transport was published. + } else if (c->disconnected_) { + GRPC_SUBCHANNEL_WEAK_UNREF(c, "connecting"); + } else { + gpr_log(GPR_INFO, "Connect failed: %s", grpc_error_string(error)); + c->SetConnectivityStateLocked(GRPC_CHANNEL_TRANSIENT_FAILURE, + "connect_failed"); + grpc_connectivity_state_set(&c->state_and_health_tracker_, + GRPC_CHANNEL_TRANSIENT_FAILURE, "connect_failed"); - grpc_connectivity_state_set(&c->state_and_health_tracker_, - GRPC_CHANNEL_TRANSIENT_FAILURE, - "connect_failed"); - c->MaybeStartConnectingLocked(); - GRPC_SUBCHANNEL_WEAK_UNREF(c, "connecting"); + c->MaybeStartConnectingLocked(); + GRPC_SUBCHANNEL_WEAK_UNREF(c, "connecting"); + } } - gpr_mu_unlock(&c->mu_); GRPC_SUBCHANNEL_WEAK_UNREF(c, "on_connecting_finished"); grpc_channel_args_destroy(delete_channel_args); } diff --git a/src/core/ext/filters/client_channel/subchannel.h b/src/core/ext/filters/client_channel/subchannel.h index 54bd13b6065..9c2e57d3e05 100644 --- a/src/core/ext/filters/client_channel/subchannel.h +++ b/src/core/ext/filters/client_channel/subchannel.h @@ -29,6 +29,7 @@ #include "src/core/lib/gpr/arena.h" #include "src/core/lib/gprpp/ref_counted.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" +#include "src/core/lib/gprpp/sync.h" #include "src/core/lib/iomgr/polling_entity.h" #include "src/core/lib/iomgr/timer.h" #include "src/core/lib/transport/connectivity_state.h" @@ -263,7 +264,7 @@ class Subchannel { // pollset_set tracking who's interested in a connection being setup. grpc_pollset_set* pollset_set_; // Protects the other members. - gpr_mu mu_; + Mutex mu_; // Refcount // - lower INTERNAL_REF_BITS bits are for internal references: // these do not keep the subchannel open. diff --git a/src/core/lib/channel/channelz_registry.cc b/src/core/lib/channel/channelz_registry.cc index 9f0169aeaab..b6a660b18fd 100644 --- a/src/core/lib/channel/channelz_registry.cc +++ b/src/core/lib/channel/channelz_registry.cc @@ -23,7 +23,7 @@ #include "src/core/lib/channel/channelz_registry.h" #include "src/core/lib/gpr/useful.h" #include "src/core/lib/gprpp/memory.h" -#include "src/core/lib/gprpp/mutex_lock.h" +#include "src/core/lib/gprpp/sync.h" #include #include diff --git a/src/core/lib/channel/handshaker.h b/src/core/lib/channel/handshaker.h index 912d524c8db..b68799b6e0e 100644 --- a/src/core/lib/channel/handshaker.h +++ b/src/core/lib/channel/handshaker.h @@ -27,8 +27,8 @@ #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/gprpp/inlined_vector.h" -#include "src/core/lib/gprpp/mutex_lock.h" #include "src/core/lib/gprpp/ref_counted.h" +#include "src/core/lib/gprpp/sync.h" #include "src/core/lib/iomgr/closure.h" #include "src/core/lib/iomgr/endpoint.h" #include "src/core/lib/iomgr/exec_ctx.h" diff --git a/src/core/lib/gprpp/mutex_lock.h b/src/core/lib/gprpp/mutex_lock.h deleted file mode 100644 index 54751d5fe46..00000000000 --- a/src/core/lib/gprpp/mutex_lock.h +++ /dev/null @@ -1,42 +0,0 @@ -/* - * - * Copyright 2018 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#ifndef GRPC_CORE_LIB_GPRPP_MUTEX_LOCK_H -#define GRPC_CORE_LIB_GPRPP_MUTEX_LOCK_H - -#include - -#include - -namespace grpc_core { - -class MutexLock { - public: - explicit MutexLock(gpr_mu* mu) : mu_(mu) { gpr_mu_lock(mu); } - ~MutexLock() { gpr_mu_unlock(mu_); } - - MutexLock(const MutexLock&) = delete; - MutexLock& operator=(const MutexLock&) = delete; - - private: - gpr_mu* const mu_; -}; - -} // namespace grpc_core - -#endif /* GRPC_CORE_LIB_GPRPP_MUTEX_LOCK_H */ diff --git a/src/core/lib/gprpp/sync.h b/src/core/lib/gprpp/sync.h new file mode 100644 index 00000000000..895ca60fec0 --- /dev/null +++ b/src/core/lib/gprpp/sync.h @@ -0,0 +1,126 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPC_CORE_LIB_GPRPP_SYNC_H +#define GRPC_CORE_LIB_GPRPP_SYNC_H + +#include + +#include +#include +#include +#include + +// The core library is not accessible in C++ codegen headers, and vice versa. +// Thus, we need to have duplicate headers with similar functionality. +// Make sure any change to this file is also reflected in +// include/grpcpp/impl/codegen/sync.h. +// +// Whenever possible, prefer using this file over +// since this file doesn't rely on g_core_codegen_interface and hence does not +// pay the costs of virtual function calls. + +namespace grpc_core { + +class Mutex { + public: + Mutex() { gpr_mu_init(&mu_); } + ~Mutex() { gpr_mu_destroy(&mu_); } + + Mutex(const Mutex&) = delete; + Mutex& operator=(const Mutex&) = delete; + + gpr_mu* get() { return &mu_; } + const gpr_mu* get() const { return &mu_; } + + private: + gpr_mu mu_; +}; + +// MutexLock is a std:: +class MutexLock { + public: + explicit MutexLock(Mutex* mu) : mu_(mu->get()) { gpr_mu_lock(mu_); } + explicit MutexLock(gpr_mu* mu) : mu_(mu) { gpr_mu_lock(mu_); } + ~MutexLock() { gpr_mu_unlock(mu_); } + + MutexLock(const MutexLock&) = delete; + MutexLock& operator=(const MutexLock&) = delete; + + private: + gpr_mu* const mu_; +}; + +class ReleasableMutexLock { + public: + explicit ReleasableMutexLock(Mutex* mu) : mu_(mu->get()) { gpr_mu_lock(mu_); } + explicit ReleasableMutexLock(gpr_mu* mu) : mu_(mu) { gpr_mu_lock(mu_); } + ~ReleasableMutexLock() { + if (!released_) gpr_mu_unlock(mu_); + } + + ReleasableMutexLock(const ReleasableMutexLock&) = delete; + ReleasableMutexLock& operator=(const ReleasableMutexLock&) = delete; + + void Lock() { + GPR_DEBUG_ASSERT(released_); + gpr_mu_lock(mu_); + released_ = false; + } + + void Unlock() { + GPR_DEBUG_ASSERT(!released_); + released_ = true; + gpr_mu_unlock(mu_); + } + + private: + gpr_mu* const mu_; + bool released_ = false; +}; + +class CondVar { + public: + CondVar() { gpr_cv_init(&cv_); } + ~CondVar() { gpr_cv_destroy(&cv_); } + + CondVar(const CondVar&) = delete; + CondVar& operator=(const CondVar&) = delete; + + void Signal() { gpr_cv_signal(&cv_); } + void Broadcast() { gpr_cv_broadcast(&cv_); } + + int Wait(Mutex* mu) { return Wait(mu, gpr_inf_future(GPR_CLOCK_REALTIME)); } + int Wait(Mutex* mu, const gpr_timespec& deadline) { + return gpr_cv_wait(&cv_, mu->get(), deadline); + } + + template + void WaitUntil(Mutex* mu, Predicate pred) { + while (!pred()) { + Wait(mu, gpr_inf_future(GPR_CLOCK_REALTIME)); + } + } + + private: + gpr_cv cv_; +}; + +} // namespace grpc_core + +#endif /* GRPC_CORE_LIB_GPRPP_SYNC_H */ diff --git a/src/core/lib/iomgr/ev_epollex_linux.cc b/src/core/lib/iomgr/ev_epollex_linux.cc index 01be46c9f68..c387f8359a0 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.cc +++ b/src/core/lib/iomgr/ev_epollex_linux.cc @@ -47,7 +47,7 @@ #include "src/core/lib/gpr/useful.h" #include "src/core/lib/gprpp/inlined_vector.h" #include "src/core/lib/gprpp/manual_constructor.h" -#include "src/core/lib/gprpp/mutex_lock.h" +#include "src/core/lib/gprpp/sync.h" #include "src/core/lib/iomgr/block_annotate.h" #include "src/core/lib/iomgr/iomgr_internal.h" #include "src/core/lib/iomgr/is_epollexclusive_available.h" diff --git a/src/core/lib/surface/init.cc b/src/core/lib/surface/init.cc index b67d4f12252..57d975564b1 100644 --- a/src/core/lib/surface/init.cc +++ b/src/core/lib/surface/init.cc @@ -33,7 +33,7 @@ #include "src/core/lib/debug/stats.h" #include "src/core/lib/debug/trace.h" #include "src/core/lib/gprpp/fork.h" -#include "src/core/lib/gprpp/mutex_lock.h" +#include "src/core/lib/gprpp/sync.h" #include "src/core/lib/http/parser.h" #include "src/core/lib/iomgr/call_combiner.h" #include "src/core/lib/iomgr/combiner.h" diff --git a/src/core/tsi/ssl/session_cache/ssl_session_cache.cc b/src/core/tsi/ssl/session_cache/ssl_session_cache.cc index f9184bcc34f..ba0745a2359 100644 --- a/src/core/tsi/ssl/session_cache/ssl_session_cache.cc +++ b/src/core/tsi/ssl/session_cache/ssl_session_cache.cc @@ -18,7 +18,7 @@ #include -#include "src/core/lib/gprpp/mutex_lock.h" +#include "src/core/lib/gprpp/sync.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/tsi/ssl/session_cache/ssl_session.h" #include "src/core/tsi/ssl/session_cache/ssl_session_cache.h" diff --git a/src/cpp/client/channel_cc.cc b/src/cpp/client/channel_cc.cc index db59d4d8416..2d5e74163a9 100644 --- a/src/cpp/client/channel_cc.cc +++ b/src/cpp/client/channel_cc.cc @@ -232,7 +232,7 @@ class ShutdownCallback : public grpc_experimental_completion_queue_functor { CompletionQueue* Channel::CallbackCQ() { // TODO(vjpai): Consider using a single global CQ for the default CQ // if there is no explicit per-channel CQ registered - std::lock_guard l(mu_); + grpc::internal::MutexLock l(&mu_); if (callback_cq_ == nullptr) { auto* shutdown_callback = new ShutdownCallback; callback_cq_ = new CompletionQueue(grpc_completion_queue_attributes{ diff --git a/src/cpp/client/client_context.cc b/src/cpp/client/client_context.cc index efb59c71a8c..b4fce79b99a 100644 --- a/src/cpp/client/client_context.cc +++ b/src/cpp/client/client_context.cc @@ -25,6 +25,7 @@ #include #include +#include #include #include #include @@ -84,7 +85,7 @@ void ClientContext::AddMetadata(const grpc::string& meta_key, void ClientContext::set_call(grpc_call* call, const std::shared_ptr& channel) { - std::unique_lock lock(mu_); + grpc::internal::MutexLock lock(&mu_); GPR_ASSERT(call_ == nullptr); call_ = call; channel_ = channel; @@ -114,7 +115,7 @@ void ClientContext::set_compression_algorithm( } void ClientContext::TryCancel() { - std::unique_lock lock(mu_); + grpc::internal::MutexLock lock(&mu_); if (call_) { SendCancelToInterceptors(); grpc_call_cancel(call_, nullptr); diff --git a/src/cpp/server/dynamic_thread_pool.cc b/src/cpp/server/dynamic_thread_pool.cc index ef99d6459f7..c8bdbdea7e6 100644 --- a/src/cpp/server/dynamic_thread_pool.cc +++ b/src/cpp/server/dynamic_thread_pool.cc @@ -21,6 +21,7 @@ #include #include +#include #include "src/core/lib/gprpp/thd.h" @@ -40,27 +41,27 @@ DynamicThreadPool::DynamicThread::~DynamicThread() { thd_.Join(); } void DynamicThreadPool::DynamicThread::ThreadFunc() { pool_->ThreadFunc(); // Now that we have killed ourselves, we should reduce the thread count - std::unique_lock lock(pool_->mu_); + grpc_core::MutexLock lock(&pool_->mu_); pool_->nthreads_--; // Move ourselves to dead list pool_->dead_threads_.push_back(this); if ((pool_->shutdown_) && (pool_->nthreads_ == 0)) { - pool_->shutdown_cv_.notify_one(); + pool_->shutdown_cv_.Signal(); } } void DynamicThreadPool::ThreadFunc() { for (;;) { // Wait until work is available or we are shutting down. - std::unique_lock lock(mu_); + grpc_core::ReleasableMutexLock lock(&mu_); if (!shutdown_ && callbacks_.empty()) { // If there are too many threads waiting, then quit this thread if (threads_waiting_ >= reserve_threads_) { break; } threads_waiting_++; - cv_.wait(lock); + cv_.Wait(&mu_); threads_waiting_--; } // Drain callbacks before considering shutdown to ensure all work @@ -68,7 +69,7 @@ void DynamicThreadPool::ThreadFunc() { if (!callbacks_.empty()) { auto cb = callbacks_.front(); callbacks_.pop(); - lock.unlock(); + lock.Unlock(); cb(); } else if (shutdown_) { break; @@ -82,7 +83,7 @@ DynamicThreadPool::DynamicThreadPool(int reserve_threads) nthreads_(0), threads_waiting_(0) { for (int i = 0; i < reserve_threads_; i++) { - std::lock_guard lock(mu_); + grpc_core::MutexLock lock(&mu_); nthreads_++; new DynamicThread(this); } @@ -95,17 +96,17 @@ void DynamicThreadPool::ReapThreads(std::list* tlist) { } DynamicThreadPool::~DynamicThreadPool() { - std::unique_lock lock(mu_); + grpc_core::MutexLock lock(&mu_); shutdown_ = true; - cv_.notify_all(); + cv_.Broadcast(); while (nthreads_ != 0) { - shutdown_cv_.wait(lock); + shutdown_cv_.Wait(&mu_); } ReapThreads(&dead_threads_); } void DynamicThreadPool::Add(const std::function& callback) { - std::lock_guard lock(mu_); + grpc_core::MutexLock lock(&mu_); // Add works to the callbacks list callbacks_.push(callback); // Increase pool size or notify as needed @@ -114,7 +115,7 @@ void DynamicThreadPool::Add(const std::function& callback) { nthreads_++; new DynamicThread(this); } else { - cv_.notify_one(); + cv_.Signal(); } // Also use this chance to harvest dead threads if (!dead_threads_.empty()) { diff --git a/src/cpp/server/dynamic_thread_pool.h b/src/cpp/server/dynamic_thread_pool.h index 5df8cf2b043..4ae0257d40b 100644 --- a/src/cpp/server/dynamic_thread_pool.h +++ b/src/cpp/server/dynamic_thread_pool.h @@ -27,6 +27,7 @@ #include +#include "src/core/lib/gprpp/sync.h" #include "src/core/lib/gprpp/thd.h" #include "src/cpp/server/thread_pool_interface.h" @@ -50,9 +51,9 @@ class DynamicThreadPool final : public ThreadPoolInterface { grpc_core::Thread thd_; void ThreadFunc(); }; - std::mutex mu_; - std::condition_variable cv_; - std::condition_variable shutdown_cv_; + grpc_core::Mutex mu_; + grpc_core::CondVar cv_; + grpc_core::CondVar shutdown_cv_; bool shutdown_; std::queue> callbacks_; int reserve_threads_; diff --git a/src/cpp/server/health/default_health_check_service.cc b/src/cpp/server/health/default_health_check_service.cc index 44aebd2f9d9..01bc51aa213 100644 --- a/src/cpp/server/health/default_health_check_service.cc +++ b/src/cpp/server/health/default_health_check_service.cc @@ -41,7 +41,7 @@ DefaultHealthCheckService::DefaultHealthCheckService() { void DefaultHealthCheckService::SetServingStatus( const grpc::string& service_name, bool serving) { - std::unique_lock lock(mu_); + grpc_core::MutexLock lock(&mu_); if (shutdown_) { // Set to NOT_SERVING in case service_name is not in the map. serving = false; @@ -51,7 +51,7 @@ void DefaultHealthCheckService::SetServingStatus( void DefaultHealthCheckService::SetServingStatus(bool serving) { const ServingStatus status = serving ? SERVING : NOT_SERVING; - std::unique_lock lock(mu_); + grpc_core::MutexLock lock(&mu_); if (shutdown_) { return; } @@ -62,7 +62,7 @@ void DefaultHealthCheckService::SetServingStatus(bool serving) { } void DefaultHealthCheckService::Shutdown() { - std::unique_lock lock(mu_); + grpc_core::MutexLock lock(&mu_); if (shutdown_) { return; } @@ -76,7 +76,7 @@ void DefaultHealthCheckService::Shutdown() { DefaultHealthCheckService::ServingStatus DefaultHealthCheckService::GetServingStatus( const grpc::string& service_name) const { - std::lock_guard lock(mu_); + grpc_core::MutexLock lock(&mu_); auto it = services_map_.find(service_name); if (it == services_map_.end()) { return NOT_FOUND; @@ -88,7 +88,7 @@ DefaultHealthCheckService::GetServingStatus( void DefaultHealthCheckService::RegisterCallHandler( const grpc::string& service_name, std::shared_ptr handler) { - std::unique_lock lock(mu_); + grpc_core::MutexLock lock(&mu_); ServiceData& service_data = services_map_[service_name]; service_data.AddCallHandler(handler /* copies ref */); HealthCheckServiceImpl::CallHandler* h = handler.get(); @@ -98,7 +98,7 @@ void DefaultHealthCheckService::RegisterCallHandler( void DefaultHealthCheckService::UnregisterCallHandler( const grpc::string& service_name, const std::shared_ptr& handler) { - std::unique_lock lock(mu_); + grpc_core::MutexLock lock(&mu_); auto it = services_map_.find(service_name); if (it == services_map_.end()) return; ServiceData& service_data = it->second; @@ -166,7 +166,7 @@ DefaultHealthCheckService::HealthCheckServiceImpl::~HealthCheckServiceImpl() { // We will reach here after the server starts shutting down. shutdown_ = true; { - std::unique_lock lock(cq_shutdown_mu_); + grpc_core::MutexLock lock(&cq_shutdown_mu_); cq_->Shutdown(); } thread_->Join(); @@ -266,7 +266,7 @@ void DefaultHealthCheckService::HealthCheckServiceImpl::CheckCallHandler:: std::make_shared(cq, database, service); CheckCallHandler* handler = static_cast(self.get()); { - std::unique_lock lock(service->cq_shutdown_mu_); + grpc_core::MutexLock lock(&service->cq_shutdown_mu_); if (service->shutdown_) return; // Request a Check() call. handler->next_ = @@ -311,7 +311,7 @@ void DefaultHealthCheckService::HealthCheckServiceImpl::CheckCallHandler:: } // Send response. { - std::unique_lock lock(service_->cq_shutdown_mu_); + grpc_core::MutexLock lock(&service_->cq_shutdown_mu_); if (!service_->shutdown_) { next_ = CallableTag(std::bind(&CheckCallHandler::OnFinishDone, this, @@ -347,7 +347,7 @@ void DefaultHealthCheckService::HealthCheckServiceImpl::WatchCallHandler:: std::make_shared(cq, database, service); WatchCallHandler* handler = static_cast(self.get()); { - std::unique_lock lock(service->cq_shutdown_mu_); + grpc_core::MutexLock lock(&service->cq_shutdown_mu_); if (service->shutdown_) return; // Request AsyncNotifyWhenDone(). handler->on_done_notified_ = @@ -402,7 +402,7 @@ void DefaultHealthCheckService::HealthCheckServiceImpl::WatchCallHandler:: void DefaultHealthCheckService::HealthCheckServiceImpl::WatchCallHandler:: SendHealth(std::shared_ptr self, ServingStatus status) { - std::unique_lock lock(send_mu_); + grpc_core::MutexLock lock(&send_mu_); // If there's already a send in flight, cache the new status, and // we'll start a new send for it when the one in flight completes. if (send_in_flight_) { @@ -420,7 +420,7 @@ void DefaultHealthCheckService::HealthCheckServiceImpl::WatchCallHandler:: ByteBuffer response; bool success = service_->EncodeResponse(status, &response); // Grab shutdown lock and send response. - std::unique_lock cq_lock(service_->cq_shutdown_mu_); + grpc_core::MutexLock cq_lock(&service_->cq_shutdown_mu_); if (service_->shutdown_) { SendFinishLocked(std::move(self), Status::CANCELLED); return; @@ -442,7 +442,7 @@ void DefaultHealthCheckService::HealthCheckServiceImpl::WatchCallHandler:: SendFinish(std::move(self), Status::CANCELLED); return; } - std::unique_lock lock(send_mu_); + grpc_core::MutexLock lock(&send_mu_); send_in_flight_ = false; // If we got a new status since we started the last send, start a // new send for it. @@ -456,7 +456,7 @@ void DefaultHealthCheckService::HealthCheckServiceImpl::WatchCallHandler:: void DefaultHealthCheckService::HealthCheckServiceImpl::WatchCallHandler:: SendFinish(std::shared_ptr self, const Status& status) { if (finish_called_) return; - std::unique_lock cq_lock(service_->cq_shutdown_mu_); + grpc_core::MutexLock cq_lock(&service_->cq_shutdown_mu_); if (service_->shutdown_) return; SendFinishLocked(std::move(self), status); } diff --git a/src/cpp/server/health/default_health_check_service.h b/src/cpp/server/health/default_health_check_service.h index 9551cd2e2cf..4b926efdfe8 100644 --- a/src/cpp/server/health/default_health_check_service.h +++ b/src/cpp/server/health/default_health_check_service.h @@ -31,6 +31,7 @@ #include #include +#include "src/core/lib/gprpp/sync.h" #include "src/core/lib/gprpp/thd.h" namespace grpc { @@ -197,7 +198,7 @@ class DefaultHealthCheckService final : public HealthCheckServiceInterface { GenericServerAsyncWriter stream_; ServerContext ctx_; - std::mutex send_mu_; + grpc_core::Mutex send_mu_; bool send_in_flight_ = false; // Guarded by mu_. ServingStatus pending_status_ = NOT_FOUND; // Guarded by mu_. @@ -226,7 +227,7 @@ class DefaultHealthCheckService final : public HealthCheckServiceInterface { // To synchronize the operations related to shutdown state of cq_, so that // we don't enqueue new tags into cq_ after it is already shut down. - std::mutex cq_shutdown_mu_; + grpc_core::Mutex cq_shutdown_mu_; std::atomic_bool shutdown_{false}; std::unique_ptr<::grpc_core::Thread> thread_; }; @@ -273,7 +274,7 @@ class DefaultHealthCheckService final : public HealthCheckServiceInterface { const grpc::string& service_name, const std::shared_ptr& handler); - mutable std::mutex mu_; + mutable grpc_core::Mutex mu_; bool shutdown_ = false; // Guarded by mu_. std::map services_map_; // Guarded by mu_. std::unique_ptr impl_; diff --git a/src/cpp/server/load_reporter/load_reporter.cc b/src/cpp/server/load_reporter/load_reporter.cc index 464063a13ff..422ea62efd5 100644 --- a/src/cpp/server/load_reporter/load_reporter.cc +++ b/src/cpp/server/load_reporter/load_reporter.cc @@ -239,7 +239,7 @@ grpc::string LoadReporter::GenerateLbId() { ::grpc::lb::v1::LoadBalancingFeedback LoadReporter::GenerateLoadBalancingFeedback() { - std::unique_lock lock(feedback_mu_); + grpc_core::ReleasableMutexLock lock(&feedback_mu_); auto now = std::chrono::system_clock::now(); // Discard records outside the window until there is only one record // outside the window, which is used as the base for difference. @@ -277,7 +277,7 @@ LoadReporter::GenerateLoadBalancingFeedback() { double cpu_limit = newest->cpu_limit - oldest->cpu_limit; std::chrono::duration duration_seconds = newest->end_time - oldest->end_time; - lock.unlock(); + lock.Unlock(); ::grpc::lb::v1::LoadBalancingFeedback feedback; feedback.set_server_utilization(static_cast(cpu_usage / cpu_limit)); feedback.set_calls_per_second( @@ -290,7 +290,7 @@ LoadReporter::GenerateLoadBalancingFeedback() { ::google::protobuf::RepeatedPtrField<::grpc::lb::v1::Load> LoadReporter::GenerateLoads(const grpc::string& hostname, const grpc::string& lb_id) { - std::lock_guard lock(store_mu_); + grpc_core::MutexLock lock(&store_mu_); auto assigned_stores = load_data_store_.GetAssignedStores(hostname, lb_id); GPR_ASSERT(assigned_stores != nullptr); GPR_ASSERT(!assigned_stores->empty()); @@ -371,7 +371,7 @@ void LoadReporter::AppendNewFeedbackRecord(uint64_t rpcs, uint64_t errors) { // This will make the load balancing feedback generation a no-op. cpu_stats = {0, 0}; } - std::unique_lock lock(feedback_mu_); + grpc_core::MutexLock lock(&feedback_mu_); feedback_records_.emplace_back(std::chrono::system_clock::now(), rpcs, errors, cpu_stats.first, cpu_stats.second); } @@ -379,7 +379,7 @@ void LoadReporter::AppendNewFeedbackRecord(uint64_t rpcs, uint64_t errors) { void LoadReporter::ReportStreamCreated(const grpc::string& hostname, const grpc::string& lb_id, const grpc::string& load_key) { - std::lock_guard lock(store_mu_); + grpc_core::MutexLock lock(&store_mu_); load_data_store_.ReportStreamCreated(hostname, lb_id, load_key); gpr_log(GPR_INFO, "[LR %p] Report stream created (host: %s, LB ID: %s, load key: %s).", @@ -388,7 +388,7 @@ void LoadReporter::ReportStreamCreated(const grpc::string& hostname, void LoadReporter::ReportStreamClosed(const grpc::string& hostname, const grpc::string& lb_id) { - std::lock_guard lock(store_mu_); + grpc_core::MutexLock lock(&store_mu_); load_data_store_.ReportStreamClosed(hostname, lb_id); gpr_log(GPR_INFO, "[LR %p] Report stream closed (host: %s, LB ID: %s).", this, hostname.c_str(), lb_id.c_str()); @@ -407,7 +407,7 @@ void LoadReporter::ProcessViewDataCallStart( LoadRecordKey key(client_ip_and_token, user_id); LoadRecordValue value = LoadRecordValue(start_count); { - std::unique_lock lock(store_mu_); + grpc_core::MutexLock lock(&store_mu_); load_data_store_.MergeRow(host, key, value); } } @@ -459,7 +459,7 @@ void LoadReporter::ProcessViewDataCallEnd( LoadRecordValue value = LoadRecordValue( 0, ok_count, error_count, bytes_sent, bytes_received, latency_ms); { - std::unique_lock lock(store_mu_); + grpc_core::MutexLock lock(&store_mu_); load_data_store_.MergeRow(host, key, value); } } @@ -486,7 +486,7 @@ void LoadReporter::ProcessViewDataOtherCallMetrics( LoadRecordValue value = LoadRecordValue( metric_name, static_cast(num_calls), total_metric_value); { - std::unique_lock lock(store_mu_); + grpc_core::MutexLock lock(&store_mu_); load_data_store_.MergeRow(host, key, value); } } diff --git a/src/cpp/server/load_reporter/load_reporter.h b/src/cpp/server/load_reporter/load_reporter.h index b2254d56016..766e02a407a 100644 --- a/src/cpp/server/load_reporter/load_reporter.h +++ b/src/cpp/server/load_reporter/load_reporter.h @@ -29,6 +29,7 @@ #include #include +#include "src/core/lib/gprpp/sync.h" #include "src/cpp/server/load_reporter/load_data_store.h" #include "src/proto/grpc/lb/v1/load_reporter.grpc.pb.h" @@ -212,11 +213,11 @@ class LoadReporter { std::atomic next_lb_id_{0}; const std::chrono::seconds feedback_sample_window_seconds_; - std::mutex feedback_mu_; + grpc_core::Mutex feedback_mu_; std::deque feedback_records_; // TODO(juanlishen): Lock in finer grain. Locking the whole store may be // too expensive. - std::mutex store_mu_; + grpc_core::Mutex store_mu_; LoadDataStore load_data_store_; std::unique_ptr census_view_provider_; std::unique_ptr cpu_stats_provider_; diff --git a/src/cpp/server/load_reporter/load_reporter_async_service_impl.cc b/src/cpp/server/load_reporter/load_reporter_async_service_impl.cc index 859ad9946c8..9eaab5d6366 100644 --- a/src/cpp/server/load_reporter/load_reporter_async_service_impl.cc +++ b/src/cpp/server/load_reporter/load_reporter_async_service_impl.cc @@ -48,7 +48,7 @@ LoadReporterAsyncServiceImpl::~LoadReporterAsyncServiceImpl() { // We will reach here after the server starts shutting down. shutdown_ = true; { - std::unique_lock lock(cq_shutdown_mu_); + grpc_core::MutexLock lock(&cq_shutdown_mu_); cq_->Shutdown(); } if (next_fetch_and_sample_alarm_ != nullptr) @@ -62,7 +62,7 @@ void LoadReporterAsyncServiceImpl::ScheduleNextFetchAndSample() { gpr_time_from_millis(kFetchAndSampleIntervalSeconds * 1000, GPR_TIMESPAN)); { - std::unique_lock lock(cq_shutdown_mu_); + grpc_core::MutexLock lock(&cq_shutdown_mu_); if (shutdown_) return; // TODO(juanlishen): Improve the Alarm implementation to reuse a single // instance for multiple events. @@ -119,7 +119,7 @@ void LoadReporterAsyncServiceImpl::ReportLoadHandler::CreateAndStart( std::make_shared(cq, service, load_reporter); ReportLoadHandler* p = handler.get(); { - std::unique_lock lock(service->cq_shutdown_mu_); + grpc_core::MutexLock lock(&service->cq_shutdown_mu_); if (service->shutdown_) return; p->on_done_notified_ = CallableTag(std::bind(&ReportLoadHandler::OnDoneNotified, p, @@ -164,9 +164,9 @@ void LoadReporterAsyncServiceImpl::ReportLoadHandler::OnRequestDelivered( // instance will deallocate itself when it's done. CreateAndStart(cq_, service_, load_reporter_); { - std::unique_lock lock(service_->cq_shutdown_mu_); + grpc_core::ReleasableMutexLock lock(&service_->cq_shutdown_mu_); if (service_->shutdown_) { - lock.release()->unlock(); + lock.Unlock(); Shutdown(std::move(self), "OnRequestDelivered"); return; } @@ -222,9 +222,9 @@ void LoadReporterAsyncServiceImpl::ReportLoadHandler::OnReadDone( SendReport(self, true /* ok */); // Expect this read to fail. { - std::unique_lock lock(service_->cq_shutdown_mu_); + grpc_core::ReleasableMutexLock lock(&service_->cq_shutdown_mu_); if (service_->shutdown_) { - lock.release()->unlock(); + lock.Unlock(); Shutdown(std::move(self), "OnReadDone"); return; } @@ -254,9 +254,9 @@ void LoadReporterAsyncServiceImpl::ReportLoadHandler::ScheduleNextReport( gpr_now(GPR_CLOCK_MONOTONIC), gpr_time_from_millis(load_report_interval_ms_, GPR_TIMESPAN)); { - std::unique_lock lock(service_->cq_shutdown_mu_); + grpc_core::ReleasableMutexLock lock(&service_->cq_shutdown_mu_); if (service_->shutdown_) { - lock.release()->unlock(); + lock.Unlock(); Shutdown(std::move(self), "ScheduleNextReport"); return; } @@ -294,9 +294,9 @@ void LoadReporterAsyncServiceImpl::ReportLoadHandler::SendReport( call_status_ = INITIAL_RESPONSE_SENT; } { - std::unique_lock lock(service_->cq_shutdown_mu_); + grpc_core::ReleasableMutexLock lock(&service_->cq_shutdown_mu_); if (service_->shutdown_) { - lock.release()->unlock(); + lock.Unlock(); Shutdown(std::move(self), "SendReport"); return; } @@ -342,7 +342,7 @@ void LoadReporterAsyncServiceImpl::ReportLoadHandler::Shutdown( // OnRequestDelivered() may be called after OnDoneNotified(), so we need to // try to Finish() every time we are in Shutdown(). if (call_status_ >= DELIVERED && call_status_ < FINISH_CALLED) { - std::unique_lock lock(service_->cq_shutdown_mu_); + grpc_core::MutexLock lock(&service_->cq_shutdown_mu_); if (!service_->shutdown_) { on_finish_done_ = CallableTag(std::bind(&ReportLoadHandler::OnFinishDone, this, diff --git a/src/cpp/server/load_reporter/load_reporter_async_service_impl.h b/src/cpp/server/load_reporter/load_reporter_async_service_impl.h index 6fc577ff493..3087cbfc04d 100644 --- a/src/cpp/server/load_reporter/load_reporter_async_service_impl.h +++ b/src/cpp/server/load_reporter/load_reporter_async_service_impl.h @@ -25,6 +25,7 @@ #include #include +#include "src/core/lib/gprpp/sync.h" #include "src/core/lib/gprpp/thd.h" #include "src/cpp/server/load_reporter/load_reporter.h" @@ -181,7 +182,7 @@ class LoadReporterAsyncServiceImpl std::unique_ptr cq_; // To synchronize the operations related to shutdown state of cq_, so that we // don't enqueue new tags into cq_ after it is already shut down. - std::mutex cq_shutdown_mu_; + grpc_core::Mutex cq_shutdown_mu_; std::atomic_bool shutdown_{false}; std::unique_ptr<::grpc_core::Thread> thread_; std::unique_ptr load_reporter_; diff --git a/src/cpp/server/server_cc.cc b/src/cpp/server/server_cc.cc index aa9fdac9bea..64a6de97d7e 100644 --- a/src/cpp/server/server_cc.cc +++ b/src/cpp/server/server_cc.cc @@ -388,9 +388,9 @@ class Server::CallbackRequest final : public Server::CallbackRequestBase { // The counter of outstanding requests must be decremented // under a lock in case it causes the server shutdown. - std::lock_guard l(server_->callback_reqs_mu_); + grpc::internal::MutexLock l(&server_->callback_reqs_mu_); if (--server_->callback_reqs_outstanding_ == 0) { - server_->callback_reqs_done_cv_.notify_one(); + server_->callback_reqs_done_cv_.Signal(); } } @@ -814,12 +814,12 @@ Server::Server( Server::~Server() { { - std::unique_lock lock(mu_); + grpc::internal::ReleasableMutexLock lock(&mu_); if (callback_cq_ != nullptr) { callback_cq_->Shutdown(); } if (started_ && !shutdown_) { - lock.unlock(); + lock.Unlock(); Shutdown(); } else if (!started_) { // Shutdown the completion queues @@ -1051,7 +1051,7 @@ void Server::Start(ServerCompletionQueue** cqs, size_t num_cqs) { } void Server::ShutdownInternal(gpr_timespec deadline) { - std::unique_lock lock(mu_); + grpc::internal::MutexLock lock(&mu_); if (shutdown_) { return; } @@ -1102,9 +1102,9 @@ void Server::ShutdownInternal(gpr_timespec deadline) { // will report a failure, indicating a shutdown and again we won't end // up incrementing the counter. { - std::unique_lock cblock(callback_reqs_mu_); - callback_reqs_done_cv_.wait( - cblock, [this] { return callback_reqs_outstanding_ == 0; }); + grpc::internal::MutexLock cblock(&callback_reqs_mu_); + callback_reqs_done_cv_.WaitUntil( + &callback_reqs_mu_, [this] { return callback_reqs_outstanding_ == 0; }); } // Drain the shutdown queue (if the previous call to AsyncNext() timed out @@ -1114,13 +1114,13 @@ void Server::ShutdownInternal(gpr_timespec deadline) { } shutdown_notified_ = true; - shutdown_cv_.notify_all(); + shutdown_cv_.Broadcast(); } void Server::Wait() { - std::unique_lock lock(mu_); + grpc::internal::MutexLock lock(&mu_); while (started_ && !shutdown_notified_) { - shutdown_cv_.wait(lock); + shutdown_cv_.Wait(&mu_); } } @@ -1322,7 +1322,7 @@ class ShutdownCallback : public grpc_experimental_completion_queue_functor { CompletionQueue* Server::CallbackCQ() { // TODO(vjpai): Consider using a single global CQ for the default CQ // if there is no explicit per-server CQ registered - std::lock_guard l(mu_); + grpc::internal::MutexLock l(&mu_); if (callback_cq_ == nullptr) { auto* shutdown_callback = new ShutdownCallback; callback_cq_ = new CompletionQueue(grpc_completion_queue_attributes{ diff --git a/src/cpp/server/server_context.cc b/src/cpp/server/server_context.cc index 2bbf0e727a1..eced89d1a79 100644 --- a/src/cpp/server/server_context.cc +++ b/src/cpp/server/server_context.cc @@ -33,6 +33,7 @@ #include #include "src/core/lib/gprpp/ref_counted.h" +#include "src/core/lib/gprpp/sync.h" #include "src/core/lib/surface/call.h" namespace grpc { @@ -96,7 +97,7 @@ class ServerContext::CompletionOp final : public internal::CallOpSetInterface { } void SetCancelCallback(std::function callback) { - std::lock_guard lock(mu_); + grpc_core::MutexLock lock(&mu_); if (finalized_ && (cancelled_ != 0)) { callback(); @@ -107,7 +108,7 @@ class ServerContext::CompletionOp final : public internal::CallOpSetInterface { } void ClearCancelCallback() { - std::lock_guard g(mu_); + grpc_core::MutexLock g(&mu_); cancel_callback_ = nullptr; } @@ -144,7 +145,7 @@ class ServerContext::CompletionOp final : public internal::CallOpSetInterface { private: bool CheckCancelledNoPluck() { - std::lock_guard g(mu_); + grpc_core::MutexLock lock(&mu_); return finalized_ ? (cancelled_ != 0) : false; } @@ -154,7 +155,7 @@ class ServerContext::CompletionOp final : public internal::CallOpSetInterface { void* tag_; void* core_cq_tag_; grpc_core::RefCount refs_; - std::mutex mu_; + grpc_core::Mutex mu_; bool finalized_; int cancelled_; // This is an int (not bool) because it is passed to core std::function cancel_callback_; @@ -186,7 +187,7 @@ void ServerContext::CompletionOp::FillOps(internal::Call* call) { bool ServerContext::CompletionOp::FinalizeResult(void** tag, bool* status) { bool ret = false; - std::unique_lock lock(mu_); + grpc_core::ReleasableMutexLock lock(&mu_); if (done_intercepting_) { /* We are done intercepting. */ if (has_tag_) { @@ -218,14 +219,12 @@ bool ServerContext::CompletionOp::FinalizeResult(void** tag, bool* status) { cancel_callback_(); } - // Release the lock since we are going to be calling a callback and - // interceptors now - lock.unlock(); + // Release the lock since we may call a callback and interceptors now. + lock.Unlock(); if (call_cancel && reactor_ != nullptr) { reactor_->MaybeCallOnCancel(); } - /* Add interception point and run through interceptors */ interceptor_methods_.AddInterceptionHookPoint( experimental::InterceptionHookPoints::POST_RECV_CLOSE); diff --git a/src/cpp/thread_manager/thread_manager.cc b/src/cpp/thread_manager/thread_manager.cc index 3e8606a76fd..2b65352f797 100644 --- a/src/cpp/thread_manager/thread_manager.cc +++ b/src/cpp/thread_manager/thread_manager.cc @@ -62,7 +62,7 @@ ThreadManager::ThreadManager(const char* name, ThreadManager::~ThreadManager() { { - std::lock_guard lock(mu_); + grpc_core::MutexLock lock(&mu_); GPR_ASSERT(num_threads_ == 0); } @@ -72,38 +72,38 @@ ThreadManager::~ThreadManager() { } void ThreadManager::Wait() { - std::unique_lock lock(mu_); + grpc_core::MutexLock lock(&mu_); while (num_threads_ != 0) { - shutdown_cv_.wait(lock); + shutdown_cv_.Wait(&mu_); } } void ThreadManager::Shutdown() { - std::lock_guard lock(mu_); + grpc_core::MutexLock lock(&mu_); shutdown_ = true; } bool ThreadManager::IsShutdown() { - std::lock_guard lock(mu_); + grpc_core::MutexLock lock(&mu_); return shutdown_; } int ThreadManager::GetMaxActiveThreadsSoFar() { - std::lock_guard list_lock(list_mu_); + grpc_core::MutexLock list_lock(&list_mu_); return max_active_threads_sofar_; } void ThreadManager::MarkAsCompleted(WorkerThread* thd) { { - std::lock_guard list_lock(list_mu_); + grpc_core::MutexLock list_lock(&list_mu_); completed_threads_.push_back(thd); } { - std::lock_guard lock(mu_); + grpc_core::MutexLock lock(&mu_); num_threads_--; if (num_threads_ == 0) { - shutdown_cv_.notify_one(); + shutdown_cv_.Signal(); } } @@ -116,7 +116,7 @@ void ThreadManager::CleanupCompletedThreads() { { // swap out the completed threads list: allows other threads to clean up // more quickly - std::unique_lock lock(list_mu_); + grpc_core::MutexLock lock(&list_mu_); completed_threads.swap(completed_threads_); } for (auto thd : completed_threads) delete thd; @@ -132,7 +132,7 @@ void ThreadManager::Initialize() { } { - std::unique_lock lock(mu_); + grpc_core::MutexLock lock(&mu_); num_pollers_ = min_pollers_; num_threads_ = min_pollers_; max_active_threads_sofar_ = min_pollers_; @@ -149,7 +149,7 @@ void ThreadManager::MainWorkLoop() { bool ok; WorkStatus work_status = PollForWork(&tag, &ok); - std::unique_lock lock(mu_); + grpc_core::ReleasableMutexLock lock(&mu_); // Reduce the number of pollers by 1 and check what happened with the poll num_pollers_--; bool done = false; @@ -176,30 +176,30 @@ void ThreadManager::MainWorkLoop() { max_active_threads_sofar_ = num_threads_; } // Drop lock before spawning thread to avoid contention - lock.unlock(); + lock.Unlock(); new WorkerThread(this); } else if (num_pollers_ > 0) { // There is still at least some thread polling, so we can go on // even though we are below the number of pollers that we would // like to have (min_pollers_) - lock.unlock(); + lock.Unlock(); } else { // There are no pollers to spare and we couldn't allocate // a new thread, so resources are exhausted! - lock.unlock(); + lock.Unlock(); resource_exhausted = true; } } else { // There are a sufficient number of pollers available so we can do // the work and continue polling with our existing poller threads - lock.unlock(); + lock.Unlock(); } // Lock is always released at this point - do the application work // or return resource exhausted if there is new work but we couldn't // get a thread in which to do it. DoWork(tag, ok, !resource_exhausted); // Take the lock again to check post conditions - lock.lock(); + lock.Lock(); // If we're shutdown, we should finish at this point. if (shutdown_) done = true; break; diff --git a/src/cpp/thread_manager/thread_manager.h b/src/cpp/thread_manager/thread_manager.h index 6f0bd17c5fe..2fbf309d421 100644 --- a/src/cpp/thread_manager/thread_manager.h +++ b/src/cpp/thread_manager/thread_manager.h @@ -26,6 +26,7 @@ #include +#include "src/core/lib/gprpp/sync.h" #include "src/core/lib/gprpp/thd.h" #include "src/core/lib/iomgr/resource_quota.h" @@ -140,10 +141,10 @@ class ThreadManager { // Protects shutdown_, num_pollers_, num_threads_ and // max_active_threads_sofar_ - std::mutex mu_; + grpc_core::Mutex mu_; bool shutdown_; - std::condition_variable shutdown_cv_; + grpc_core::CondVar shutdown_cv_; // The resource user object to use when requesting quota to create threads // @@ -169,7 +170,7 @@ class ThreadManager { // ever set so far int max_active_threads_sofar_; - std::mutex list_mu_; + grpc_core::Mutex list_mu_; std::list completed_threads_; }; diff --git a/test/cpp/client/client_channel_stress_test.cc b/test/cpp/client/client_channel_stress_test.cc index 91419cb257b..d326b2ed37e 100644 --- a/test/cpp/client/client_channel_stress_test.cc +++ b/test/cpp/client/client_channel_stress_test.cc @@ -31,6 +31,7 @@ #include #include #include +#include #include #include @@ -168,24 +169,24 @@ class ClientChannelStressTest { explicit ServerThread(const grpc::string& type, const grpc::string& server_host, T* service) : type_(type), service_(service) { - std::mutex mu; + grpc::internal::Mutex mu; // We need to acquire the lock here in order to prevent the notify_one // by ServerThread::Start from firing before the wait below is hit. - std::unique_lock lock(mu); + grpc::internal::MutexLock lock(&mu); port_ = grpc_pick_unused_port_or_die(); gpr_log(GPR_INFO, "starting %s server on port %d", type_.c_str(), port_); - std::condition_variable cond; + grpc::internal::CondVar cond; thread_.reset(new std::thread( std::bind(&ServerThread::Start, this, server_host, &mu, &cond))); - cond.wait(lock); + cond.Wait(&mu); gpr_log(GPR_INFO, "%s server startup complete", type_.c_str()); } - void Start(const grpc::string& server_host, std::mutex* mu, - std::condition_variable* cond) { + void Start(const grpc::string& server_host, grpc::internal::Mutex* mu, + grpc::internal::CondVar* cond) { // We need to acquire the lock here in order to prevent the notify_one // below from firing before its corresponding wait is executed. - std::lock_guard lock(*mu); + grpc::internal::MutexLock lock(mu); std::ostringstream server_address; server_address << server_host << ":" << port_; ServerBuilder builder; @@ -193,7 +194,7 @@ class ClientChannelStressTest { InsecureServerCredentials()); builder.RegisterService(service_); server_ = builder.BuildAndStart(); - cond->notify_one(); + cond->Signal(); } void Shutdown() { diff --git a/test/cpp/end2end/client_lb_end2end_test.cc b/test/cpp/end2end/client_lb_end2end_test.cc index 77f9db94acc..6623a2ff55f 100644 --- a/test/cpp/end2end/client_lb_end2end_test.cc +++ b/test/cpp/end2end/client_lb_end2end_test.cc @@ -33,6 +33,7 @@ #include #include #include +#include #include #include @@ -98,7 +99,7 @@ class MyTestServiceImpl : public TestServiceImpl { Status Echo(ServerContext* context, const EchoRequest* request, EchoResponse* response) override { { - std::unique_lock lock(mu_); + grpc::internal::MutexLock lock(&mu_); ++request_count_; } AddClient(context->peer()); @@ -106,29 +107,29 @@ class MyTestServiceImpl : public TestServiceImpl { } int request_count() { - std::unique_lock lock(mu_); + grpc::internal::MutexLock lock(&mu_); return request_count_; } void ResetCounters() { - std::unique_lock lock(mu_); + grpc::internal::MutexLock lock(&mu_); request_count_ = 0; } std::set clients() { - std::unique_lock lock(clients_mu_); + grpc::internal::MutexLock lock(&clients_mu_); return clients_; } private: void AddClient(const grpc::string& client) { - std::unique_lock lock(clients_mu_); + grpc::internal::MutexLock lock(&clients_mu_); clients_.insert(client); } - std::mutex mu_; + grpc::internal::Mutex mu_; int request_count_; - std::mutex clients_mu_; + grpc::internal::Mutex clients_mu_; std::set clients_; }; @@ -293,18 +294,18 @@ class ClientLbEnd2endTest : public ::testing::Test { void Start(const grpc::string& server_host) { gpr_log(GPR_INFO, "starting server on port %d", port_); started_ = true; - std::mutex mu; - std::unique_lock lock(mu); - std::condition_variable cond; + grpc::internal::Mutex mu; + grpc::internal::MutexLock lock(&mu); + grpc::internal::CondVar cond; thread_.reset(new std::thread( std::bind(&ServerData::Serve, this, server_host, &mu, &cond))); - cond.wait(lock, [this] { return server_ready_; }); + cond.WaitUntil(&mu, [this] { return server_ready_; }); server_ready_ = false; gpr_log(GPR_INFO, "server startup complete"); } - void Serve(const grpc::string& server_host, std::mutex* mu, - std::condition_variable* cond) { + void Serve(const grpc::string& server_host, grpc::internal::Mutex* mu, + grpc::internal::CondVar* cond) { std::ostringstream server_address; server_address << server_host << ":" << port_; ServerBuilder builder; @@ -313,9 +314,9 @@ class ClientLbEnd2endTest : public ::testing::Test { builder.AddListeningPort(server_address.str(), std::move(creds)); builder.RegisterService(&service_); server_ = builder.BuildAndStart(); - std::lock_guard lock(*mu); + grpc::internal::MutexLock lock(mu); server_ready_ = true; - cond->notify_one(); + cond->Signal(); } void Shutdown() { @@ -1374,7 +1375,7 @@ class ClientLbInterceptTrailingMetadataTest : public ClientLbEnd2endTest { void TearDown() override { ClientLbEnd2endTest::TearDown(); } int trailers_intercepted() { - std::unique_lock lock(mu_); + grpc::internal::MutexLock lock(&mu_); return trailers_intercepted_; } @@ -1382,11 +1383,11 @@ class ClientLbInterceptTrailingMetadataTest : public ClientLbEnd2endTest { static void ReportTrailerIntercepted(void* arg) { ClientLbInterceptTrailingMetadataTest* self = static_cast(arg); - std::unique_lock lock(self->mu_); + grpc::internal::MutexLock lock(&self->mu_); self->trailers_intercepted_++; } - std::mutex mu_; + grpc::internal::Mutex mu_; int trailers_intercepted_ = 0; }; diff --git a/test/cpp/end2end/grpclb_end2end_test.cc b/test/cpp/end2end/grpclb_end2end_test.cc index 0dc3746c2d7..f8d887dd24d 100644 --- a/test/cpp/end2end/grpclb_end2end_test.cc +++ b/test/cpp/end2end/grpclb_end2end_test.cc @@ -30,6 +30,7 @@ #include #include #include +#include #include #include @@ -85,32 +86,32 @@ template class CountedService : public ServiceType { public: size_t request_count() { - std::unique_lock lock(mu_); + grpc::internal::MutexLock lock(&mu_); return request_count_; } size_t response_count() { - std::unique_lock lock(mu_); + grpc::internal::MutexLock lock(&mu_); return response_count_; } void IncreaseResponseCount() { - std::unique_lock lock(mu_); + grpc::internal::MutexLock lock(&mu_); ++response_count_; } void IncreaseRequestCount() { - std::unique_lock lock(mu_); + grpc::internal::MutexLock lock(&mu_); ++request_count_; } void ResetCounters() { - std::unique_lock lock(mu_); + grpc::internal::MutexLock lock(&mu_); request_count_ = 0; response_count_ = 0; } protected: - std::mutex mu_; + grpc::internal::Mutex mu_; private: size_t request_count_ = 0; @@ -148,18 +149,18 @@ class BackendServiceImpl : public BackendService { void Shutdown() {} std::set clients() { - std::unique_lock lock(clients_mu_); + grpc::internal::MutexLock lock(&clients_mu_); return clients_; } private: void AddClient(const grpc::string& client) { - std::unique_lock lock(clients_mu_); + grpc::internal::MutexLock lock(&clients_mu_); clients_.insert(client); } - std::mutex mu_; - std::mutex clients_mu_; + grpc::internal::Mutex mu_; + grpc::internal::Mutex clients_mu_; std::set clients_; }; @@ -210,7 +211,7 @@ class BalancerServiceImpl : public BalancerService { Status BalanceLoad(ServerContext* context, Stream* stream) override { gpr_log(GPR_INFO, "LB[%p]: BalanceLoad", this); { - std::unique_lock lock(mu_); + grpc::internal::MutexLock lock(&mu_); if (serverlist_done_) goto done; } { @@ -237,7 +238,7 @@ class BalancerServiceImpl : public BalancerService { } { - std::unique_lock lock(mu_); + grpc::internal::MutexLock lock(&mu_); responses_and_delays = responses_and_delays_; } for (const auto& response_and_delay : responses_and_delays) { @@ -245,8 +246,8 @@ class BalancerServiceImpl : public BalancerService { response_and_delay.second); } { - std::unique_lock lock(mu_); - serverlist_cond_.wait(lock, [this] { return serverlist_done_; }); + grpc::internal::MutexLock lock(&mu_); + serverlist_cond_.WaitUntil(&mu_, [this] { return serverlist_done_; }); } if (client_load_reporting_interval_seconds_ > 0) { @@ -257,7 +258,7 @@ class BalancerServiceImpl : public BalancerService { GPR_ASSERT(request.has_client_stats()); // We need to acquire the lock here in order to prevent the notify_one // below from firing before its corresponding wait is executed. - std::lock_guard lock(mu_); + grpc::internal::MutexLock lock(&mu_); client_stats_.num_calls_started += request.client_stats().num_calls_started(); client_stats_.num_calls_finished += @@ -274,7 +275,7 @@ class BalancerServiceImpl : public BalancerService { drop_token_count.num_calls(); } load_report_ready_ = true; - load_report_cond_.notify_one(); + load_report_cond_.Signal(); } } } @@ -284,12 +285,12 @@ class BalancerServiceImpl : public BalancerService { } void add_response(const LoadBalanceResponse& response, int send_after_ms) { - std::unique_lock lock(mu_); + grpc::internal::MutexLock lock(&mu_); responses_and_delays_.push_back(std::make_pair(response, send_after_ms)); } void Start() { - std::lock_guard lock(mu_); + grpc::internal::MutexLock lock(&mu_); serverlist_done_ = false; load_report_ready_ = false; responses_and_delays_.clear(); @@ -326,17 +327,17 @@ class BalancerServiceImpl : public BalancerService { } const ClientStats& WaitForLoadReport() { - std::unique_lock lock(mu_); - load_report_cond_.wait(lock, [this] { return load_report_ready_; }); + grpc::internal::MutexLock lock(&mu_); + load_report_cond_.WaitUntil(&mu_, [this] { return load_report_ready_; }); load_report_ready_ = false; return client_stats_; } void NotifyDoneWithServerlists() { - std::lock_guard lock(mu_); + grpc::internal::MutexLock lock(&mu_); if (!serverlist_done_) { serverlist_done_ = true; - serverlist_cond_.notify_all(); + serverlist_cond_.Broadcast(); } } @@ -355,10 +356,10 @@ class BalancerServiceImpl : public BalancerService { const int client_load_reporting_interval_seconds_; std::vector responses_and_delays_; - std::mutex mu_; - std::condition_variable load_report_cond_; + grpc::internal::Mutex mu_; + grpc::internal::CondVar load_report_cond_; bool load_report_ready_ = false; - std::condition_variable serverlist_cond_; + grpc::internal::CondVar serverlist_cond_; bool serverlist_done_ = false; ClientStats client_stats_; }; @@ -624,22 +625,22 @@ class GrpclbEnd2endTest : public ::testing::Test { GPR_ASSERT(!running_); running_ = true; service_.Start(); - std::mutex mu; + grpc::internal::Mutex mu; // We need to acquire the lock here in order to prevent the notify_one // by ServerThread::Serve from firing before the wait below is hit. - std::unique_lock lock(mu); - std::condition_variable cond; + grpc::internal::MutexLock lock(&mu); + grpc::internal::CondVar cond; thread_.reset(new std::thread( std::bind(&ServerThread::Serve, this, server_host, &mu, &cond))); - cond.wait(lock); + cond.Wait(&mu); gpr_log(GPR_INFO, "%s server startup complete", type_.c_str()); } - void Serve(const grpc::string& server_host, std::mutex* mu, - std::condition_variable* cond) { + void Serve(const grpc::string& server_host, grpc::internal::Mutex* mu, + grpc::internal::CondVar* cond) { // We need to acquire the lock here in order to prevent the notify_one // below from firing before its corresponding wait is executed. - std::lock_guard lock(*mu); + grpc::internal::MutexLock lock(mu); std::ostringstream server_address; server_address << server_host << ":" << port_; ServerBuilder builder; @@ -648,7 +649,7 @@ class GrpclbEnd2endTest : public ::testing::Test { builder.AddListeningPort(server_address.str(), creds); builder.RegisterService(&service_); server_ = builder.BuildAndStart(); - cond->notify_one(); + cond->Signal(); } void Shutdown() { diff --git a/test/cpp/end2end/thread_stress_test.cc b/test/cpp/end2end/thread_stress_test.cc index e30ce0dbcbf..5b8af61ee33 100644 --- a/test/cpp/end2end/thread_stress_test.cc +++ b/test/cpp/end2end/thread_stress_test.cc @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -188,7 +189,7 @@ class CommonStressTestAsyncServer : public BaseClass { } void TearDown() override { { - std::unique_lock l(mu_); + grpc::internal::MutexLock l(&mu_); this->TearDownStart(); shutting_down_ = true; cq_->Shutdown(); @@ -229,7 +230,7 @@ class CommonStressTestAsyncServer : public BaseClass { } } void RefreshContext(int i) { - std::unique_lock l(mu_); + grpc::internal::MutexLock l(&mu_); if (!shutting_down_) { contexts_[i].state = Context::READY; contexts_[i].srv_ctx.reset(new ServerContext); @@ -253,7 +254,7 @@ class CommonStressTestAsyncServer : public BaseClass { ::grpc::testing::EchoTestService::AsyncService service_; std::unique_ptr cq_; bool shutting_down_; - std::mutex mu_; + grpc::internal::Mutex mu_; std::vector server_threads_; }; @@ -341,9 +342,9 @@ class AsyncClientEnd2endTest : public ::testing::Test { } void Wait() { - std::unique_lock l(mu_); + grpc::internal::MutexLock l(&mu_); while (rpcs_outstanding_ != 0) { - cv_.wait(l); + cv_.Wait(&mu_); } cq_.Shutdown(); @@ -366,7 +367,7 @@ class AsyncClientEnd2endTest : public ::testing::Test { call->response_reader->Finish(&call->response, &call->status, (void*)call); - std::unique_lock l(mu_); + grpc::internal::MutexLock l(&mu_); rpcs_outstanding_++; } } @@ -384,20 +385,20 @@ class AsyncClientEnd2endTest : public ::testing::Test { bool notify; { - std::unique_lock l(mu_); + grpc::internal::MutexLock l(&mu_); rpcs_outstanding_--; notify = (rpcs_outstanding_ == 0); } if (notify) { - cv_.notify_all(); + cv_.Signal(); } } } Common common_; CompletionQueue cq_; - std::mutex mu_; - std::condition_variable cv_; + grpc::internal::Mutex mu_; + grpc::internal::CondVar cv_; int rpcs_outstanding_; }; diff --git a/test/cpp/end2end/xds_end2end_test.cc b/test/cpp/end2end/xds_end2end_test.cc index c767a4485ce..ee248239909 100644 --- a/test/cpp/end2end/xds_end2end_test.cc +++ b/test/cpp/end2end/xds_end2end_test.cc @@ -84,32 +84,32 @@ template class CountedService : public ServiceType { public: size_t request_count() { - std::unique_lock lock(mu_); + grpc::internal::MutexLock lock(&mu_); return request_count_; } size_t response_count() { - std::unique_lock lock(mu_); + grpc::internal::MutexLock lock(&mu_); return response_count_; } void IncreaseResponseCount() { - std::unique_lock lock(mu_); + grpc::internal::MutexLock lock(&mu_); ++response_count_; } void IncreaseRequestCount() { - std::unique_lock lock(mu_); + grpc::internal::MutexLock lock(&mu_); ++request_count_; } void ResetCounters() { - std::unique_lock lock(mu_); + grpc::internal::MutexLock lock(&mu_); request_count_ = 0; response_count_ = 0; } protected: - std::mutex mu_; + grpc::internal::Mutex mu_; private: size_t request_count_ = 0; @@ -145,18 +145,18 @@ class BackendServiceImpl : public BackendService { void Shutdown() {} std::set clients() { - std::unique_lock lock(clients_mu_); + grpc::internal::MutexLock lock(&clients_mu_); return clients_; } private: void AddClient(const grpc::string& client) { - std::unique_lock lock(clients_mu_); + grpc::internal::MutexLock lock(&clients_mu_); clients_.insert(client); } - std::mutex mu_; - std::mutex clients_mu_; + grpc::internal::Mutex mu_; + grpc::internal::Mutex clients_mu_; std::set clients_; }; @@ -208,7 +208,7 @@ class BalancerServiceImpl : public BalancerService { // TODO(juanlishen): Clean up the scoping. gpr_log(GPR_INFO, "LB[%p]: BalanceLoad", this); { - std::unique_lock lock(mu_); + grpc::internal::MutexLock lock(&mu_); if (serverlist_done_) goto done; } { @@ -234,7 +234,7 @@ class BalancerServiceImpl : public BalancerService { } { - std::unique_lock lock(mu_); + grpc::internal::MutexLock lock(&mu_); responses_and_delays = responses_and_delays_; } for (const auto& response_and_delay : responses_and_delays) { @@ -242,8 +242,8 @@ class BalancerServiceImpl : public BalancerService { response_and_delay.second); } { - std::unique_lock lock(mu_); - serverlist_cond_.wait(lock, [this] { return serverlist_done_; }); + grpc::internal::MutexLock lock(&mu_); + serverlist_cond_.WaitUntil(&mu_, [this] { return serverlist_done_; }); } if (client_load_reporting_interval_seconds_ > 0) { @@ -254,7 +254,7 @@ class BalancerServiceImpl : public BalancerService { GPR_ASSERT(request.has_client_stats()); // We need to acquire the lock here in order to prevent the notify_one // below from firing before its corresponding wait is executed. - std::lock_guard lock(mu_); + grpc::internal::MutexLock lock(&mu_); client_stats_.num_calls_started += request.client_stats().num_calls_started(); client_stats_.num_calls_finished += @@ -271,7 +271,7 @@ class BalancerServiceImpl : public BalancerService { drop_token_count.num_calls(); } load_report_ready_ = true; - load_report_cond_.notify_one(); + load_report_cond_.Signal(); } } } @@ -281,12 +281,12 @@ class BalancerServiceImpl : public BalancerService { } void add_response(const LoadBalanceResponse& response, int send_after_ms) { - std::unique_lock lock(mu_); + grpc::internal::MutexLock lock(&mu_); responses_and_delays_.push_back(std::make_pair(response, send_after_ms)); } void Shutdown() { - std::unique_lock lock(mu_); + grpc::internal::MutexLock lock(&mu_); NotifyDoneWithServerlistsLocked(); responses_and_delays_.clear(); client_stats_.Reset(); @@ -318,21 +318,21 @@ class BalancerServiceImpl : public BalancerService { } const ClientStats& WaitForLoadReport() { - std::unique_lock lock(mu_); - load_report_cond_.wait(lock, [this] { return load_report_ready_; }); + grpc::internal::MutexLock lock(&mu_); + load_report_cond_.WaitUntil(&mu_, [this] { return load_report_ready_; }); load_report_ready_ = false; return client_stats_; } void NotifyDoneWithServerlists() { - std::lock_guard lock(mu_); + grpc::internal::MutexLock lock(&mu_); NotifyDoneWithServerlistsLocked(); } void NotifyDoneWithServerlistsLocked() { if (!serverlist_done_) { serverlist_done_ = true; - serverlist_cond_.notify_all(); + serverlist_cond_.Broadcast(); } } @@ -351,10 +351,10 @@ class BalancerServiceImpl : public BalancerService { const int client_load_reporting_interval_seconds_; std::vector responses_and_delays_; - std::mutex mu_; - std::condition_variable load_report_cond_; + grpc::internal::Mutex mu_; + grpc::internal::CondVar load_report_cond_; bool load_report_ready_ = false; - std::condition_variable serverlist_cond_; + grpc::internal::CondVar serverlist_cond_; bool serverlist_done_ = false; ClientStats client_stats_; }; @@ -637,22 +637,22 @@ class XdsEnd2endTest : public ::testing::Test { gpr_log(GPR_INFO, "starting %s server on port %d", type_.c_str(), port_); GPR_ASSERT(!running_); running_ = true; - std::mutex mu; + grpc::internal::Mutex mu; // We need to acquire the lock here in order to prevent the notify_one // by ServerThread::Serve from firing before the wait below is hit. - std::unique_lock lock(mu); - std::condition_variable cond; + grpc::internal::MutexLock lock(&mu); + grpc::internal::CondVar cond; thread_.reset(new std::thread( std::bind(&ServerThread::Serve, this, server_host, &mu, &cond))); - cond.wait(lock); + cond.Wait(&mu); gpr_log(GPR_INFO, "%s server startup complete", type_.c_str()); } - void Serve(const grpc::string& server_host, std::mutex* mu, - std::condition_variable* cond) { + void Serve(const grpc::string& server_host, grpc::internal::Mutex* mu, + grpc::internal::CondVar* cond) { // We need to acquire the lock here in order to prevent the notify_one // below from firing before its corresponding wait is executed. - std::lock_guard lock(*mu); + grpc::internal::MutexLock lock(mu); std::ostringstream server_address; server_address << server_host << ":" << port_; ServerBuilder builder; @@ -661,7 +661,7 @@ class XdsEnd2endTest : public ::testing::Test { builder.AddListeningPort(server_address.str(), creds); builder.RegisterService(&service_); server_ = builder.BuildAndStart(); - cond->notify_one(); + cond->Signal(); } void Shutdown() { diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 7274f9588b3..a0a6e952409 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -987,6 +987,7 @@ include/grpcpp/impl/codegen/status.h \ include/grpcpp/impl/codegen/status_code_enum.h \ include/grpcpp/impl/codegen/string_ref.h \ include/grpcpp/impl/codegen/stub_options.h \ +include/grpcpp/impl/codegen/sync.h \ include/grpcpp/impl/codegen/sync_stream.h \ include/grpcpp/impl/codegen/time.h \ include/grpcpp/impl/grpc_library.h \ diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 80760da0c58..c48c15bd1f5 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -989,6 +989,7 @@ include/grpcpp/impl/codegen/status.h \ include/grpcpp/impl/codegen/status_code_enum.h \ include/grpcpp/impl/codegen/string_ref.h \ include/grpcpp/impl/codegen/stub_options.h \ +include/grpcpp/impl/codegen/sync.h \ include/grpcpp/impl/codegen/sync_stream.h \ include/grpcpp/impl/codegen/time.h \ include/grpcpp/impl/grpc_library.h \ @@ -1086,12 +1087,12 @@ src/core/lib/gprpp/inlined_vector.h \ src/core/lib/gprpp/manual_constructor.h \ src/core/lib/gprpp/map.h \ src/core/lib/gprpp/memory.h \ -src/core/lib/gprpp/mutex_lock.h \ src/core/lib/gprpp/optional.h \ src/core/lib/gprpp/orphanable.h \ src/core/lib/gprpp/pair.h \ src/core/lib/gprpp/ref_counted.h \ src/core/lib/gprpp/ref_counted_ptr.h \ +src/core/lib/gprpp/sync.h \ src/core/lib/gprpp/thd.h \ src/core/lib/http/format_request.h \ src/core/lib/http/httpcli.h \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 8827e4b7e38..dad6c98269d 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1168,12 +1168,12 @@ src/core/lib/gprpp/inlined_vector.h \ src/core/lib/gprpp/manual_constructor.h \ src/core/lib/gprpp/map.h \ src/core/lib/gprpp/memory.h \ -src/core/lib/gprpp/mutex_lock.h \ src/core/lib/gprpp/optional.h \ src/core/lib/gprpp/orphanable.h \ src/core/lib/gprpp/pair.h \ src/core/lib/gprpp/ref_counted.h \ src/core/lib/gprpp/ref_counted_ptr.h \ +src/core/lib/gprpp/sync.h \ src/core/lib/gprpp/thd.h \ src/core/lib/gprpp/thd_posix.cc \ src/core/lib/gprpp/thd_windows.cc \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index b41d10b61f3..bd6d68bc5c3 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -8050,8 +8050,8 @@ "src/core/lib/gprpp/manual_constructor.h", "src/core/lib/gprpp/map.h", "src/core/lib/gprpp/memory.h", - "src/core/lib/gprpp/mutex_lock.h", "src/core/lib/gprpp/pair.h", + "src/core/lib/gprpp/sync.h", "src/core/lib/gprpp/thd.h", "src/core/lib/profiling/timers.h" ], @@ -8098,8 +8098,8 @@ "src/core/lib/gprpp/manual_constructor.h", "src/core/lib/gprpp/map.h", "src/core/lib/gprpp/memory.h", - "src/core/lib/gprpp/mutex_lock.h", "src/core/lib/gprpp/pair.h", + "src/core/lib/gprpp/sync.h", "src/core/lib/gprpp/thd.h", "src/core/lib/profiling/timers.h" ], @@ -9880,6 +9880,7 @@ }, { "deps": [ + "grpc++_internal_hdrs_only", "grpc_codegen" ], "headers": [ @@ -10076,6 +10077,7 @@ "gpr", "gpr_base_headers", "grpc++_codegen_base", + "grpc++_internal_hdrs_only", "grpc_base_headers", "grpc_transport_inproc_headers", "health_proto", @@ -10370,6 +10372,20 @@ "third_party": false, "type": "filegroup" }, + { + "deps": [], + "headers": [ + "include/grpcpp/impl/codegen/sync.h" + ], + "is_filegroup": true, + "language": "c++", + "name": "grpc++_internal_hdrs_only", + "src": [ + "include/grpcpp/impl/codegen/sync.h" + ], + "third_party": false, + "type": "filegroup" + }, { "deps": [], "headers": [ From 1c186784cd011d758cac79249dade3460e81cb96 Mon Sep 17 00:00:00 2001 From: Soheil Hassas Yeganeh Date: Tue, 16 Apr 2019 14:54:04 -0400 Subject: [PATCH 08/13] Make sure grpc::internal::Mutex has enough space for gpr_mu, std::mutex, and pthread_mutex_t. --- include/grpc/impl/codegen/port_platform.h | 10 ++++++++++ include/grpcpp/impl/codegen/sync.h | 17 +++++++++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/include/grpc/impl/codegen/port_platform.h b/include/grpc/impl/codegen/port_platform.h index 0349e31bb3b..d7294d59d41 100644 --- a/include/grpc/impl/codegen/port_platform.h +++ b/include/grpc/impl/codegen/port_platform.h @@ -115,6 +115,7 @@ #define GPR_POSIX_SUBPROCESS 1 #define GPR_POSIX_SYNC 1 #define GPR_POSIX_TIME 1 +#define GPR_HAS_PTHREAD_H 1 #define GPR_GETPID_IN_UNISTD_H 1 #ifdef _LP64 #define GPR_ARCH_64 1 @@ -144,6 +145,7 @@ #define GPR_POSIX_SUBPROCESS 1 #define GPR_POSIX_SYNC 1 #define GPR_POSIX_TIME 1 +#define GPR_HAS_PTHREAD_H 1 #define GPR_GETPID_IN_UNISTD_H 1 #define GPR_SUPPORT_CHANNELS_FROM_FD 1 #elif defined(__linux__) @@ -170,6 +172,7 @@ #define GPR_POSIX_SUBPROCESS 1 #define GPR_POSIX_SYNC 1 #define GPR_POSIX_TIME 1 +#define GPR_HAS_PTHREAD_H 1 #define GPR_GETPID_IN_UNISTD_H 1 #ifdef _LP64 #define GPR_ARCH_64 1 @@ -235,6 +238,7 @@ #define GPR_POSIX_SUBPROCESS 1 #define GPR_POSIX_SYNC 1 #define GPR_POSIX_TIME 1 +#define GPR_HAS_PTHREAD_H 1 #define GPR_GETPID_IN_UNISTD_H 1 #ifndef GRPC_CFSTREAM #define GPR_SUPPORT_CHANNELS_FROM_FD 1 @@ -260,6 +264,7 @@ #define GPR_POSIX_SUBPROCESS 1 #define GPR_POSIX_SYNC 1 #define GPR_POSIX_TIME 1 +#define GPR_HAS_PTHREAD_H 1 #define GPR_GETPID_IN_UNISTD_H 1 #define GPR_SUPPORT_CHANNELS_FROM_FD 1 #ifdef _LP64 @@ -283,6 +288,7 @@ #define GPR_POSIX_SUBPROCESS 1 #define GPR_POSIX_SYNC 1 #define GPR_POSIX_TIME 1 +#define GPR_HAS_PTHREAD_H 1 #define GPR_GETPID_IN_UNISTD_H 1 #define GPR_SUPPORT_CHANNELS_FROM_FD 1 #ifdef _LP64 @@ -303,6 +309,7 @@ #define GPR_POSIX_SUBPROCESS 1 #define GPR_POSIX_SYNC 1 #define GPR_POSIX_TIME 1 +#define GPR_HAS_PTHREAD_H 1 #define GPR_GETPID_IN_UNISTD_H 1 #ifdef _LP64 #define GPR_ARCH_64 1 @@ -325,6 +332,7 @@ #define GPR_POSIX_SUBPROCESS 1 #define GPR_POSIX_SYNC 1 #define GPR_POSIX_TIME 1 +#define GPR_HAS_PTHREAD_H 1 #define GPR_GETPID_IN_UNISTD_H 1 #ifdef _LP64 #define GPR_ARCH_64 1 @@ -353,6 +361,7 @@ #define GPR_POSIX_SUBPROCESS 1 #define GPR_POSIX_SYNC 1 #define GPR_POSIX_TIME 1 +#define GPR_HAS_PTHREAD_H 1 #define GPR_GETPID_IN_UNISTD_H 1 #ifdef _LP64 #define GPR_ARCH_64 1 @@ -378,6 +387,7 @@ #define GPR_POSIX_SYNC 1 #define GPR_POSIX_STRING 1 #define GPR_POSIX_TIME 1 +#define GPR_HAS_PTHREAD_H 1 #define GPR_GETPID_IN_UNISTD_H 1 #else #error "Could not auto-detect platform" diff --git a/include/grpcpp/impl/codegen/sync.h b/include/grpcpp/impl/codegen/sync.h index 2ed71eeb9f2..146f182e57b 100644 --- a/include/grpcpp/impl/codegen/sync.h +++ b/include/grpcpp/impl/codegen/sync.h @@ -19,8 +19,15 @@ #ifndef GRPCPP_IMPL_CODEGEN_SYNC_H #define GRPCPP_IMPL_CODEGEN_SYNC_H -#include #include + +#ifdef GPR_HAS_PTHREAD_H +#include +#endif + +#include + +#include #include #include @@ -49,7 +56,13 @@ class Mutex { const gpr_mu* get() const { return &mu_; } private: - gpr_mu mu_; + union { + gpr_mu mu_; + std::mutex do_not_use_sth_; +#ifdef GPR_HAS_PTHREAD_H + pthread_mutex_t do_not_use_pth_; +#endif + }; }; // MutexLock is a std:: From 0d9b4212f887dda4f8bfd0ec71917a7063c01bb0 Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Wed, 17 Apr 2019 11:41:22 -0700 Subject: [PATCH 09/13] Cleanup Clang Tidy errors --- src/core/ext/filters/client_channel/client_channel.cc | 4 ++-- src/core/lib/compression/compression_args.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/ext/filters/client_channel/client_channel.cc b/src/core/ext/filters/client_channel/client_channel.cc index 0304a265a09..689978a4b32 100644 --- a/src/core/ext/filters/client_channel/client_channel.cc +++ b/src/core/ext/filters/client_channel/client_channel.cc @@ -146,7 +146,7 @@ class ChannelData { return picker_.get(); } void AddQueuedPick(QueuedPick* pick, grpc_polling_entity* pollent); - void RemoveQueuedPick(QueuedPick* pick, grpc_polling_entity* pollent); + void RemoveQueuedPick(QueuedPick* to_remove, grpc_polling_entity* pollent); bool received_service_config_data() const { return received_service_config_data_; @@ -223,7 +223,7 @@ class ChannelData { ~ChannelData(); static bool ProcessResolverResultLocked( - void* arg, Resolver::Result* args, const char** lb_policy_name, + void* arg, Resolver::Result* result, const char** lb_policy_name, RefCountedPtr* lb_policy_config); grpc_error* DoPingLocked(grpc_transport_op* op); diff --git a/src/core/lib/compression/compression_args.h b/src/core/lib/compression/compression_args.h index 75084ab0d91..407d6e2b8ca 100644 --- a/src/core/lib/compression/compression_args.h +++ b/src/core/lib/compression/compression_args.h @@ -42,7 +42,7 @@ grpc_channel_args* grpc_channel_args_set_compression_algorithm( * modified to point to the returned instance (which may be different from the * input value of \a a). */ grpc_channel_args* grpc_channel_args_compression_algorithm_set_state( - grpc_channel_args** a, grpc_compression_algorithm algorithm, int enabled); + grpc_channel_args** a, grpc_compression_algorithm algorithm, int state); /** Returns the bitset representing the support state (true for enabled, false * for disabled) for compression algorithms. From 212e4b865734f31603e94cafa2fbf8b9774573a5 Mon Sep 17 00:00:00 2001 From: Soheil Hassas Yeganeh Date: Wed, 17 Apr 2019 15:26:07 -0400 Subject: [PATCH 10/13] Retire the GRPC_ARENA_INIT_STRATEGY env variable. This is in the hot path and it is not needed anymore. Remove it for good. --- doc/environment_variables.md | 10 -------- src/core/lib/gpr/arena.cc | 50 ++++-------------------------------- src/core/lib/gpr/arena.h | 2 -- src/core/lib/surface/init.cc | 1 - 4 files changed, 5 insertions(+), 58 deletions(-) diff --git a/doc/environment_variables.md b/doc/environment_variables.md index e8d0dbd25f8..43dcd7616e7 100644 --- a/doc/environment_variables.md +++ b/doc/environment_variables.md @@ -145,13 +145,3 @@ some configuration as environment variables that can be set. * grpc_cfstream set to 1 to turn on CFStream experiment. With this experiment gRPC uses CFStream API to make TCP connections. The option is only available on iOS platform and when macro GRPC_CFSTREAM is defined. - -* GRPC_ARENA_INIT_STRATEGY - Selects the initialization strategy for blocks allocated in the arena. Valid - values are: - - no_init (default): Do not initialize the arena block. - - zero_init: Initialize the arena blocks with 0. - - non_zero_init: Initialize the arena blocks with a non-zero value. - - NOTE: This environment variable is experimental and will be removed. Thus, it - should not be relied upon. diff --git a/src/core/lib/gpr/arena.cc b/src/core/lib/gpr/arena.cc index 836a7ca793d..ede5cc48050 100644 --- a/src/core/lib/gpr/arena.cc +++ b/src/core/lib/gpr/arena.cc @@ -29,49 +29,10 @@ #include #include "src/core/lib/gpr/alloc.h" -#include "src/core/lib/gpr/env.h" #include "src/core/lib/gprpp/memory.h" -namespace { -enum init_strategy { - NO_INIT, // Do not initialize the arena blocks. - ZERO_INIT, // Initialize arena blocks with 0. - NON_ZERO_INIT, // Initialize arena blocks with a non-zero value. -}; - -gpr_once g_init_strategy_once = GPR_ONCE_INIT; -init_strategy g_init_strategy = NO_INIT; -} // namespace - -static void set_strategy_from_env() { - char* str = gpr_getenv("GRPC_ARENA_INIT_STRATEGY"); - if (str == nullptr) { - g_init_strategy = NO_INIT; - } else if (strcmp(str, "zero_init") == 0) { - g_init_strategy = ZERO_INIT; - } else if (strcmp(str, "non_zero_init") == 0) { - g_init_strategy = NON_ZERO_INIT; - } else { - g_init_strategy = NO_INIT; - } - gpr_free(str); -} - -static void* gpr_arena_alloc_maybe_init(size_t size) { - void* mem = gpr_malloc_aligned(size, GPR_MAX_ALIGNMENT); - gpr_once_init(&g_init_strategy_once, set_strategy_from_env); - if (GPR_UNLIKELY(g_init_strategy != NO_INIT)) { - if (g_init_strategy == ZERO_INIT) { - memset(mem, 0, size); - } else { // NON_ZERO_INIT. - memset(mem, 0xFE, size); - } - } - return mem; -} - -void gpr_arena_init() { - gpr_once_init(&g_init_strategy_once, set_strategy_from_env); +static void* gpr_arena_malloc(size_t size) { + return gpr_malloc_aligned(size, GPR_MAX_ALIGNMENT); } // Uncomment this to use a simple arena that simply allocates the @@ -109,8 +70,7 @@ void* gpr_arena_alloc(gpr_arena* arena, size_t size) { gpr_mu_lock(&arena->mu); arena->ptrs = (void**)gpr_realloc(arena->ptrs, sizeof(void*) * (arena->num_ptrs + 1)); - void* retval = arena->ptrs[arena->num_ptrs++] = - gpr_arena_alloc_maybe_init(size); + void* retval = arena->ptrs[arena->num_ptrs++] = gpr_arena_malloc(size); gpr_mu_unlock(&arena->mu); return retval; } @@ -154,7 +114,7 @@ struct gpr_arena { gpr_arena* gpr_arena_create(size_t initial_size) { initial_size = GPR_ROUND_UP_TO_ALIGNMENT_SIZE(initial_size); - return new (gpr_arena_alloc_maybe_init( + return new (gpr_arena_malloc( GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(gpr_arena)) + initial_size)) gpr_arena(initial_size); } @@ -179,7 +139,7 @@ void* gpr_arena_alloc(gpr_arena* arena, size_t size) { // sizing historesis (that is, most calls should have a large enough initial // zone and will not need to grow the arena). gpr_mu_lock(&arena->arena_growth_mutex); - zone* z = new (gpr_arena_alloc_maybe_init( + zone* z = new (gpr_arena_malloc( GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(zone)) + size)) zone(); arena->last_zone->next = z; arena->last_zone = z; diff --git a/src/core/lib/gpr/arena.h b/src/core/lib/gpr/arena.h index 069892b2282..6d2a073dd58 100644 --- a/src/core/lib/gpr/arena.h +++ b/src/core/lib/gpr/arena.h @@ -37,7 +37,5 @@ gpr_arena* gpr_arena_create(size_t initial_size); void* gpr_arena_alloc(gpr_arena* arena, size_t size); // Destroy an arena, returning the total number of bytes allocated size_t gpr_arena_destroy(gpr_arena* arena); -// Initializes the Arena component. -void gpr_arena_init(); #endif /* GRPC_CORE_LIB_GPR_ARENA_H */ diff --git a/src/core/lib/surface/init.cc b/src/core/lib/surface/init.cc index b67d4f12252..a19e1c94505 100644 --- a/src/core/lib/surface/init.cc +++ b/src/core/lib/surface/init.cc @@ -133,7 +133,6 @@ void grpc_init(void) { } grpc_core::Fork::GlobalInit(); grpc_fork_handlers_auto_register(); - gpr_arena_init(); grpc_stats_init(); grpc_slice_intern_init(); grpc_mdctx_global_init(); From 4b4ecdf3b9e931a1e26d38c1f6d1f716e4d7137e Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Wed, 17 Apr 2019 13:35:56 -0700 Subject: [PATCH 11/13] Fixed sanitize.sh It has been broken since check_copyright dropped the --fix option. This script is modified not to use the missing option. --- tools/distrib/sanitize.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/distrib/sanitize.sh b/tools/distrib/sanitize.sh index fed1e64b7ca..13488b1e235 100755 --- a/tools/distrib/sanitize.sh +++ b/tools/distrib/sanitize.sh @@ -29,11 +29,11 @@ if [ "x$1" == 'x--pre-commit' ]; then fi fi CHANGED_FILES=$(eval $DIFF_COMMAND) ./tools/distrib/clang_format_code.sh - ./tools/distrib/check_copyright.py --fix --precommit + ./tools/distrib/check_copyright.py --precommit ./tools/distrib/check_trailing_newlines.sh else ./tools/buildgen/generate_projects.sh ./tools/distrib/clang_format_code.sh - ./tools/distrib/check_copyright.py --fix + ./tools/distrib/check_copyright.py ./tools/distrib/check_trailing_newlines.sh fi From a09d705dbf6fe345016d3c411864cab05e22e4a1 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Fri, 19 Apr 2019 08:02:54 -0700 Subject: [PATCH 12/13] Convert client_channel call_data to C++. --- .../filters/client_channel/client_channel.cc | 2191 +++++++++-------- 1 file changed, 1113 insertions(+), 1078 deletions(-) diff --git a/src/core/ext/filters/client_channel/client_channel.cc b/src/core/ext/filters/client_channel/client_channel.cc index 0304a265a09..14eac3bed29 100644 --- a/src/core/ext/filters/client_channel/client_channel.cc +++ b/src/core/ext/filters/client_channel/client_channel.cc @@ -71,8 +71,6 @@ using grpc_core::internal::ClientChannelMethodParamsTable; using grpc_core::internal::ProcessedResolverResult; using grpc_core::internal::ServerRetryThrottleData; -using grpc_core::LoadBalancingPolicy; - // // Client channel filter // @@ -85,16 +83,21 @@ using grpc_core::LoadBalancingPolicy; // any even moderately compelling reason to do so. #define RETRY_BACKOFF_JITTER 0.2 -grpc_core::TraceFlag grpc_client_channel_call_trace(false, - "client_channel_call"); -grpc_core::TraceFlag grpc_client_channel_routing_trace( - false, "client_channel_routing"); - -// Forward declarations. -static void start_pick_locked(void* arg, grpc_error* error); -static void maybe_apply_service_config_to_call_locked(grpc_call_element* elem); +// Max number of batches that can be pending on a call at any given +// time. This includes one batch for each of the following ops: +// recv_initial_metadata +// send_initial_metadata +// recv_message +// send_message +// recv_trailing_metadata +// send_trailing_metadata +#define MAX_PENDING_BATCHES 6 namespace grpc_core { + +TraceFlag grpc_client_channel_call_trace(false, "client_channel_call"); +TraceFlag grpc_client_channel_routing_trace(false, "client_channel_routing"); + namespace { // @@ -278,6 +281,411 @@ class ChannelData { UniquePtr info_service_config_json_; }; +// +// CallData definition +// + +class CallData { + public: + static grpc_error* Init(grpc_call_element* elem, + const grpc_call_element_args* args); + static void Destroy(grpc_call_element* elem, + const grpc_call_final_info* final_info, + grpc_closure* then_schedule_closure); + static void StartTransportStreamOpBatch( + grpc_call_element* elem, grpc_transport_stream_op_batch* batch); + static void SetPollent(grpc_call_element* elem, grpc_polling_entity* pollent); + + RefCountedPtr subchannel_call() { return subchannel_call_; } + + // Invoked by channel for queued picks once resolver results are available. + void MaybeApplyServiceConfigToCallLocked(grpc_call_element* elem); + + // Invoked by channel for queued picks when the picker is updated. + static void StartPickLocked(void* arg, grpc_error* error); + + private: + class QueuedPickCanceller; + + // State used for starting a retryable batch on a subchannel call. + // This provides its own grpc_transport_stream_op_batch and other data + // structures needed to populate the ops in the batch. + // We allocate one struct on the arena for each attempt at starting a + // batch on a given subchannel call. + struct SubchannelCallBatchData { + // Creates a SubchannelCallBatchData object on the call's arena with the + // specified refcount. If set_on_complete is true, the batch's + // on_complete callback will be set to point to on_complete(); + // otherwise, the batch's on_complete callback will be null. + static SubchannelCallBatchData* Create(grpc_call_element* elem, + int refcount, bool set_on_complete); + + void Unref() { + if (gpr_unref(&refs)) Destroy(); + } + + SubchannelCallBatchData(grpc_call_element* elem, CallData* calld, + int refcount, bool set_on_complete); + // All dtor code must be added in `Destroy()`. This is because we may + // call closures in `SubchannelCallBatchData` after they are unrefed by + // `Unref()`, and msan would complain about accessing this class + // after calling dtor. As a result we cannot call the `dtor` in `Unref()`. + // TODO(soheil): We should try to call the dtor in `Unref()`. + ~SubchannelCallBatchData() { Destroy(); } + void Destroy(); + + gpr_refcount refs; + grpc_call_element* elem; + RefCountedPtr subchannel_call; + // The batch to use in the subchannel call. + // Its payload field points to SubchannelCallRetryState::batch_payload. + grpc_transport_stream_op_batch batch; + // For intercepting on_complete. + grpc_closure on_complete; + }; + + // Retry state associated with a subchannel call. + // Stored in the parent_data of the subchannel call object. + struct SubchannelCallRetryState { + explicit SubchannelCallRetryState(grpc_call_context_element* context) + : batch_payload(context), + started_send_initial_metadata(false), + completed_send_initial_metadata(false), + started_send_trailing_metadata(false), + completed_send_trailing_metadata(false), + started_recv_initial_metadata(false), + completed_recv_initial_metadata(false), + started_recv_trailing_metadata(false), + completed_recv_trailing_metadata(false), + retry_dispatched(false) {} + + // SubchannelCallBatchData.batch.payload points to this. + grpc_transport_stream_op_batch_payload batch_payload; + // For send_initial_metadata. + // Note that we need to make a copy of the initial metadata for each + // subchannel call instead of just referring to the copy in call_data, + // because filters in the subchannel stack will probably add entries, + // so we need to start in a pristine state for each attempt of the call. + grpc_linked_mdelem* send_initial_metadata_storage; + grpc_metadata_batch send_initial_metadata; + // For send_message. + // TODO(roth): Restructure this to eliminate use of ManualConstructor. + ManualConstructor send_message; + // For send_trailing_metadata. + grpc_linked_mdelem* send_trailing_metadata_storage; + grpc_metadata_batch send_trailing_metadata; + // For intercepting recv_initial_metadata. + grpc_metadata_batch recv_initial_metadata; + grpc_closure recv_initial_metadata_ready; + bool trailing_metadata_available = false; + // For intercepting recv_message. + grpc_closure recv_message_ready; + OrphanablePtr recv_message; + // For intercepting recv_trailing_metadata. + grpc_metadata_batch recv_trailing_metadata; + grpc_transport_stream_stats collect_stats; + grpc_closure recv_trailing_metadata_ready; + // These fields indicate which ops have been started and completed on + // this subchannel call. + size_t started_send_message_count = 0; + size_t completed_send_message_count = 0; + size_t started_recv_message_count = 0; + size_t completed_recv_message_count = 0; + bool started_send_initial_metadata : 1; + bool completed_send_initial_metadata : 1; + bool started_send_trailing_metadata : 1; + bool completed_send_trailing_metadata : 1; + bool started_recv_initial_metadata : 1; + bool completed_recv_initial_metadata : 1; + bool started_recv_trailing_metadata : 1; + bool completed_recv_trailing_metadata : 1; + // State for callback processing. + SubchannelCallBatchData* recv_initial_metadata_ready_deferred_batch = + nullptr; + grpc_error* recv_initial_metadata_error = GRPC_ERROR_NONE; + SubchannelCallBatchData* recv_message_ready_deferred_batch = nullptr; + grpc_error* recv_message_error = GRPC_ERROR_NONE; + SubchannelCallBatchData* recv_trailing_metadata_internal_batch = nullptr; + // NOTE: Do not move this next to the metadata bitfields above. That would + // save space but will also result in a data race because compiler + // will generate a 2 byte store which overwrites the meta-data + // fields upon setting this field. + bool retry_dispatched : 1; + }; + + // Pending batches stored in call data. + struct PendingBatch { + // The pending batch. If nullptr, this slot is empty. + grpc_transport_stream_op_batch* batch; + // Indicates whether payload for send ops has been cached in CallData. + bool send_ops_cached; + }; + + CallData(grpc_call_element* elem, const ChannelData& chand, + const grpc_call_element_args& args); + ~CallData(); + + // Caches data for send ops so that it can be retried later, if not + // already cached. + void MaybeCacheSendOpsForBatch(PendingBatch* pending); + void FreeCachedSendInitialMetadata(ChannelData* chand); + // Frees cached send_message at index idx. + void FreeCachedSendMessage(ChannelData* chand, size_t idx); + void FreeCachedSendTrailingMetadata(ChannelData* chand); + // Frees cached send ops that have already been completed after + // committing the call. + void FreeCachedSendOpDataAfterCommit(grpc_call_element* elem, + SubchannelCallRetryState* retry_state); + // Frees cached send ops that were completed by the completed batch in + // batch_data. Used when batches are completed after the call is committed. + void FreeCachedSendOpDataForCompletedBatch( + grpc_call_element* elem, SubchannelCallBatchData* batch_data, + SubchannelCallRetryState* retry_state); + + static void MaybeInjectRecvTrailingMetadataReadyForLoadBalancingPolicy( + const LoadBalancingPolicy::PickArgs& pick, + grpc_transport_stream_op_batch* batch); + + // Returns the index into pending_batches_ to be used for batch. + static size_t GetBatchIndex(grpc_transport_stream_op_batch* batch); + void PendingBatchesAdd(grpc_call_element* elem, + grpc_transport_stream_op_batch* batch); + void PendingBatchClear(PendingBatch* pending); + void MaybeClearPendingBatch(grpc_call_element* elem, PendingBatch* pending); + static void FailPendingBatchInCallCombiner(void* arg, grpc_error* error); + // A predicate type and some useful implementations for PendingBatchesFail(). + typedef bool (*YieldCallCombinerPredicate)( + const CallCombinerClosureList& closures); + static bool YieldCallCombiner(const CallCombinerClosureList& closures) { + return true; + } + static bool NoYieldCallCombiner(const CallCombinerClosureList& closures) { + return false; + } + static bool YieldCallCombinerIfPendingBatchesFound( + const CallCombinerClosureList& closures) { + return closures.size() > 0; + } + // Fails all pending batches. + // If yield_call_combiner_predicate returns true, assumes responsibility for + // yielding the call combiner. + void PendingBatchesFail( + grpc_call_element* elem, grpc_error* error, + YieldCallCombinerPredicate yield_call_combiner_predicate); + static void ResumePendingBatchInCallCombiner(void* arg, grpc_error* ignored); + // Resumes all pending batches on subchannel_call_. + void PendingBatchesResume(grpc_call_element* elem); + // Returns a pointer to the first pending batch for which predicate(batch) + // returns true, or null if not found. + template + PendingBatch* PendingBatchFind(grpc_call_element* elem, + const char* log_message, Predicate predicate); + + // Commits the call so that no further retry attempts will be performed. + void RetryCommit(grpc_call_element* elem, + SubchannelCallRetryState* retry_state); + // Starts a retry after appropriate back-off. + void DoRetry(grpc_call_element* elem, SubchannelCallRetryState* retry_state, + grpc_millis server_pushback_ms); + // Returns true if the call is being retried. + bool MaybeRetry(grpc_call_element* elem, SubchannelCallBatchData* batch_data, + grpc_status_code status, grpc_mdelem* server_pushback_md); + + // Invokes recv_initial_metadata_ready for a subchannel batch. + static void InvokeRecvInitialMetadataCallback(void* arg, grpc_error* error); + // Intercepts recv_initial_metadata_ready callback for retries. + // Commits the call and returns the initial metadata up the stack. + static void RecvInitialMetadataReady(void* arg, grpc_error* error); + + // Invokes recv_message_ready for a subchannel batch. + static void InvokeRecvMessageCallback(void* arg, grpc_error* error); + // Intercepts recv_message_ready callback for retries. + // Commits the call and returns the message up the stack. + static void RecvMessageReady(void* arg, grpc_error* error); + + // Sets *status and *server_pushback_md based on md_batch and error. + // Only sets *server_pushback_md if server_pushback_md != nullptr. + void GetCallStatus(grpc_call_element* elem, grpc_metadata_batch* md_batch, + grpc_error* error, grpc_status_code* status, + grpc_mdelem** server_pushback_md); + // Adds recv_trailing_metadata_ready closure to closures. + void AddClosureForRecvTrailingMetadataReady( + grpc_call_element* elem, SubchannelCallBatchData* batch_data, + grpc_error* error, CallCombinerClosureList* closures); + // Adds any necessary closures for deferred recv_initial_metadata and + // recv_message callbacks to closures. + static void AddClosuresForDeferredRecvCallbacks( + SubchannelCallBatchData* batch_data, + SubchannelCallRetryState* retry_state, CallCombinerClosureList* closures); + // Returns true if any op in the batch was not yet started. + // Only looks at send ops, since recv ops are always started immediately. + bool PendingBatchIsUnstarted(PendingBatch* pending, + SubchannelCallRetryState* retry_state); + // For any pending batch containing an op that has not yet been started, + // adds the pending batch's completion closures to closures. + void AddClosuresToFailUnstartedPendingBatches( + grpc_call_element* elem, SubchannelCallRetryState* retry_state, + grpc_error* error, CallCombinerClosureList* closures); + // Runs necessary closures upon completion of a call attempt. + void RunClosuresForCompletedCall(SubchannelCallBatchData* batch_data, + grpc_error* error); + // Intercepts recv_trailing_metadata_ready callback for retries. + // Commits the call and returns the trailing metadata up the stack. + static void RecvTrailingMetadataReady(void* arg, grpc_error* error); + + // Adds the on_complete closure for the pending batch completed in + // batch_data to closures. + void AddClosuresForCompletedPendingBatch( + grpc_call_element* elem, SubchannelCallBatchData* batch_data, + SubchannelCallRetryState* retry_state, grpc_error* error, + CallCombinerClosureList* closures); + + // If there are any cached ops to replay or pending ops to start on the + // subchannel call, adds a closure to closures to invoke + // StartRetriableSubchannelBatches(). + void AddClosuresForReplayOrPendingSendOps( + grpc_call_element* elem, SubchannelCallBatchData* batch_data, + SubchannelCallRetryState* retry_state, CallCombinerClosureList* closures); + + // Callback used to intercept on_complete from subchannel calls. + // Called only when retries are enabled. + static void OnComplete(void* arg, grpc_error* error); + + static void StartBatchInCallCombiner(void* arg, grpc_error* ignored); + // Adds a closure to closures that will execute batch in the call combiner. + void AddClosureForSubchannelBatch(grpc_call_element* elem, + grpc_transport_stream_op_batch* batch, + CallCombinerClosureList* closures); + // Adds retriable send_initial_metadata op to batch_data. + void AddRetriableSendInitialMetadataOp(SubchannelCallRetryState* retry_state, + SubchannelCallBatchData* batch_data); + // Adds retriable send_message op to batch_data. + void AddRetriableSendMessageOp(grpc_call_element* elem, + SubchannelCallRetryState* retry_state, + SubchannelCallBatchData* batch_data); + // Adds retriable send_trailing_metadata op to batch_data. + void AddRetriableSendTrailingMetadataOp(SubchannelCallRetryState* retry_state, + SubchannelCallBatchData* batch_data); + // Adds retriable recv_initial_metadata op to batch_data. + void AddRetriableRecvInitialMetadataOp(SubchannelCallRetryState* retry_state, + SubchannelCallBatchData* batch_data); + // Adds retriable recv_message op to batch_data. + void AddRetriableRecvMessageOp(SubchannelCallRetryState* retry_state, + SubchannelCallBatchData* batch_data); + // Adds retriable recv_trailing_metadata op to batch_data. + void AddRetriableRecvTrailingMetadataOp(SubchannelCallRetryState* retry_state, + SubchannelCallBatchData* batch_data); + // Helper function used to start a recv_trailing_metadata batch. This + // is used in the case where a recv_initial_metadata or recv_message + // op fails in a way that we know the call is over but when the application + // has not yet started its own recv_trailing_metadata op. + void StartInternalRecvTrailingMetadata(grpc_call_element* elem); + // If there are any cached send ops that need to be replayed on the + // current subchannel call, creates and returns a new subchannel batch + // to replay those ops. Otherwise, returns nullptr. + SubchannelCallBatchData* MaybeCreateSubchannelBatchForReplay( + grpc_call_element* elem, SubchannelCallRetryState* retry_state); + // Adds subchannel batches for pending batches to closures. + void AddSubchannelBatchesForPendingBatches( + grpc_call_element* elem, SubchannelCallRetryState* retry_state, + CallCombinerClosureList* closures); + // Constructs and starts whatever subchannel batches are needed on the + // subchannel call. + static void StartRetriableSubchannelBatches(void* arg, grpc_error* ignored); + + void CreateSubchannelCall(grpc_call_element* elem); + // Invoked when a pick is completed, on both success or failure. + static void PickDone(void* arg, grpc_error* error); + // Removes the call from the channel's list of queued picks. + void RemoveCallFromQueuedPicksLocked(grpc_call_element* elem); + // Adds the call to the channel's list of queued picks. + void AddCallToQueuedPicksLocked(grpc_call_element* elem); + // Applies service config to the call. Must be invoked once we know + // that the resolver has returned results to the channel. + void ApplyServiceConfigToCallLocked(grpc_call_element* elem); + + // State for handling deadlines. + // The code in deadline_filter.c requires this to be the first field. + // TODO(roth): This is slightly sub-optimal in that grpc_deadline_state + // and this struct both independently store pointers to the call stack + // and call combiner. If/when we have time, find a way to avoid this + // without breaking the grpc_deadline_state abstraction. + grpc_deadline_state deadline_state_; + + grpc_slice path_; // Request path. + gpr_timespec call_start_time_; + grpc_millis deadline_; + gpr_arena* arena_; + grpc_call_stack* owning_call_; + grpc_call_combiner* call_combiner_; + grpc_call_context_element* call_context_; + + RefCountedPtr retry_throttle_data_; + RefCountedPtr method_params_; + + RefCountedPtr subchannel_call_; + + // Set when we get a cancel_stream op. + grpc_error* cancel_error_ = GRPC_ERROR_NONE; + + ChannelData::QueuedPick pick_; + bool pick_queued_ = false; + bool service_config_applied_ = false; + QueuedPickCanceller* pick_canceller_ = nullptr; + grpc_closure pick_closure_; + + grpc_polling_entity* pollent_ = nullptr; + + // Batches are added to this list when received from above. + // They are removed when we are done handling the batch (i.e., when + // either we have invoked all of the batch's callbacks or we have + // passed the batch down to the subchannel call and are not + // intercepting any of its callbacks). + PendingBatch pending_batches_[MAX_PENDING_BATCHES] = {}; + bool pending_send_initial_metadata_ : 1; + bool pending_send_message_ : 1; + bool pending_send_trailing_metadata_ : 1; + + // Retry state. + bool enable_retries_ : 1; + bool retry_committed_ : 1; + bool last_attempt_got_server_pushback_ : 1; + int num_attempts_completed_ = 0; + size_t bytes_buffered_for_retry_ = 0; + // TODO(roth): Restructure this to eliminate use of ManualConstructor. + ManualConstructor retry_backoff_; + grpc_timer retry_timer_; + + // The number of pending retriable subchannel batches containing send ops. + // We hold a ref to the call stack while this is non-zero, since replay + // batches may not complete until after all callbacks have been returned + // to the surface, and we need to make sure that the call is not destroyed + // until all of these batches have completed. + // Note that we actually only need to track replay batches, but it's + // easier to track all batches with send ops. + int num_pending_retriable_subchannel_send_batches_ = 0; + + // Cached data for retrying send ops. + // send_initial_metadata + bool seen_send_initial_metadata_ = false; + grpc_linked_mdelem* send_initial_metadata_storage_ = nullptr; + grpc_metadata_batch send_initial_metadata_; + uint32_t send_initial_metadata_flags_; + gpr_atm* peer_string_; + // send_message + // When we get a send_message op, we replace the original byte stream + // with a CachingByteStream that caches the slices to a local buffer for + // use in retries. + // Note: We inline the cache for the first 3 send_message ops and use + // dynamic allocation after that. This number was essentially picked + // at random; it could be changed in the future to tune performance. + InlinedVector send_messages_; + // send_trailing_metadata + bool seen_send_trailing_metadata_ = false; + grpc_linked_mdelem* send_trailing_metadata_storage_ = nullptr; + grpc_metadata_batch send_trailing_metadata_; +}; + // // ChannelData::ConnectivityStateAndPickerSetter // @@ -333,7 +741,7 @@ class ChannelData::ConnectivityStateAndPickerSetter { // Re-process queued picks. for (QueuedPick* pick = self->chand_->queued_picks_; pick != nullptr; pick = pick->next) { - start_pick_locked(pick->elem, GRPC_ERROR_NONE); + CallData::StartPickLocked(pick->elem, GRPC_ERROR_NONE); } // Clean up. GRPC_CHANNEL_STACK_UNREF(self->chand_->owning_stack_, @@ -378,7 +786,8 @@ class ChannelData::ServiceConfigSetter { // Apply service config to queued picks. for (QueuedPick* pick = chand->queued_picks_; pick != nullptr; pick = pick->next) { - maybe_apply_service_config_to_call_locked(pick->elem); + CallData* calld = static_cast(pick->elem->call_data); + calld->MaybeApplyServiceConfigToCallLocked(pick->elem); } // Clean up. GRPC_CHANNEL_STACK_UNREF(self->chand_->owning_stack_, @@ -877,24 +1286,9 @@ grpc_connectivity_state ChannelData::CheckConnectivityState( return out; } -} // namespace -} // namespace grpc_core - -/************************************************************************* - * PER-CALL FUNCTIONS - */ - -using grpc_core::ChannelData; - -// Max number of batches that can be pending on a call at any given -// time. This includes one batch for each of the following ops: -// recv_initial_metadata -// send_initial_metadata -// recv_message -// send_message -// recv_trailing_metadata -// send_trailing_metadata -#define MAX_PENDING_BATCHES 6 +// +// CallData implementation +// // Retry support: // @@ -931,362 +1325,247 @@ using grpc_core::ChannelData; // (census filter is on top of this one) // - add census stats for retries -namespace grpc_core { -namespace { -class QueuedPickCanceller; -} // namespace -} // namespace grpc_core - -namespace { - -struct call_data; - -// State used for starting a retryable batch on a subchannel call. -// This provides its own grpc_transport_stream_op_batch and other data -// structures needed to populate the ops in the batch. -// We allocate one struct on the arena for each attempt at starting a -// batch on a given subchannel call. -struct subchannel_batch_data { - subchannel_batch_data(grpc_call_element* elem, call_data* calld, int refcount, - bool set_on_complete); - // All dtor code must be added in `destroy`. This is because we may - // call closures in `subchannel_batch_data` after they are unrefed by - // `batch_data_unref`, and msan would complain about accessing this class - // after calling dtor. As a result we cannot call the `dtor` in - // `batch_data_unref`. - // TODO(soheil): We should try to call the dtor in `batch_data_unref`. - ~subchannel_batch_data() { destroy(); } - void destroy(); - - gpr_refcount refs; - grpc_call_element* elem; - grpc_core::RefCountedPtr subchannel_call; - // The batch to use in the subchannel call. - // Its payload field points to subchannel_call_retry_state.batch_payload. - grpc_transport_stream_op_batch batch; - // For intercepting on_complete. - grpc_closure on_complete; -}; - -// Retry state associated with a subchannel call. -// Stored in the parent_data of the subchannel call object. -struct subchannel_call_retry_state { - explicit subchannel_call_retry_state(grpc_call_context_element* context) - : batch_payload(context), - started_send_initial_metadata(false), - completed_send_initial_metadata(false), - started_send_trailing_metadata(false), - completed_send_trailing_metadata(false), - started_recv_initial_metadata(false), - completed_recv_initial_metadata(false), - started_recv_trailing_metadata(false), - completed_recv_trailing_metadata(false), - retry_dispatched(false) {} - - // subchannel_batch_data.batch.payload points to this. - grpc_transport_stream_op_batch_payload batch_payload; - // For send_initial_metadata. - // Note that we need to make a copy of the initial metadata for each - // subchannel call instead of just referring to the copy in call_data, - // because filters in the subchannel stack will probably add entries, - // so we need to start in a pristine state for each attempt of the call. - grpc_linked_mdelem* send_initial_metadata_storage; - grpc_metadata_batch send_initial_metadata; - // For send_message. - grpc_core::ManualConstructor - send_message; - // For send_trailing_metadata. - grpc_linked_mdelem* send_trailing_metadata_storage; - grpc_metadata_batch send_trailing_metadata; - // For intercepting recv_initial_metadata. - grpc_metadata_batch recv_initial_metadata; - grpc_closure recv_initial_metadata_ready; - bool trailing_metadata_available = false; - // For intercepting recv_message. - grpc_closure recv_message_ready; - grpc_core::OrphanablePtr recv_message; - // For intercepting recv_trailing_metadata. - grpc_metadata_batch recv_trailing_metadata; - grpc_transport_stream_stats collect_stats; - grpc_closure recv_trailing_metadata_ready; - // These fields indicate which ops have been started and completed on - // this subchannel call. - size_t started_send_message_count = 0; - size_t completed_send_message_count = 0; - size_t started_recv_message_count = 0; - size_t completed_recv_message_count = 0; - bool started_send_initial_metadata : 1; - bool completed_send_initial_metadata : 1; - bool started_send_trailing_metadata : 1; - bool completed_send_trailing_metadata : 1; - bool started_recv_initial_metadata : 1; - bool completed_recv_initial_metadata : 1; - bool started_recv_trailing_metadata : 1; - bool completed_recv_trailing_metadata : 1; - // State for callback processing. - subchannel_batch_data* recv_initial_metadata_ready_deferred_batch = nullptr; - grpc_error* recv_initial_metadata_error = GRPC_ERROR_NONE; - subchannel_batch_data* recv_message_ready_deferred_batch = nullptr; - grpc_error* recv_message_error = GRPC_ERROR_NONE; - subchannel_batch_data* recv_trailing_metadata_internal_batch = nullptr; - // NOTE: Do not move this next to the metadata bitfields above. That would - // save space but will also result in a data race because compiler will - // generate a 2 byte store which overwrites the meta-data fields upon - // setting this field. - bool retry_dispatched : 1; -}; +CallData::CallData(grpc_call_element* elem, const ChannelData& chand, + const grpc_call_element_args& args) + : deadline_state_(elem, args.call_stack, args.call_combiner, + GPR_LIKELY(chand.deadline_checking_enabled()) + ? args.deadline + : GRPC_MILLIS_INF_FUTURE), + path_(grpc_slice_ref_internal(args.path)), + call_start_time_(args.start_time), + deadline_(args.deadline), + arena_(args.arena), + owning_call_(args.call_stack), + call_combiner_(args.call_combiner), + call_context_(args.context), + pending_send_initial_metadata_(false), + pending_send_message_(false), + pending_send_trailing_metadata_(false), + enable_retries_(chand.enable_retries()), + retry_committed_(false), + last_attempt_got_server_pushback_(false) {} + +CallData::~CallData() { + grpc_slice_unref_internal(path_); + GRPC_ERROR_UNREF(cancel_error_); + // Make sure there are no remaining pending batches. + for (size_t i = 0; i < GPR_ARRAY_SIZE(pending_batches_); ++i) { + GPR_ASSERT(pending_batches_[i].batch == nullptr); + } +} + +grpc_error* CallData::Init(grpc_call_element* elem, + const grpc_call_element_args* args) { + ChannelData* chand = static_cast(elem->channel_data); + new (elem->call_data) CallData(elem, *chand, *args); + return GRPC_ERROR_NONE; +} -// Pending batches stored in call data. -struct pending_batch { - // The pending batch. If nullptr, this slot is empty. - grpc_transport_stream_op_batch* batch; - // Indicates whether payload for send ops has been cached in call data. - bool send_ops_cached; -}; +void CallData::Destroy(grpc_call_element* elem, + const grpc_call_final_info* final_info, + grpc_closure* then_schedule_closure) { + CallData* calld = static_cast(elem->call_data); + if (GPR_LIKELY(calld->subchannel_call_ != nullptr)) { + calld->subchannel_call_->SetAfterCallStackDestroy(then_schedule_closure); + then_schedule_closure = nullptr; + } + calld->~CallData(); + GRPC_CLOSURE_SCHED(then_schedule_closure, GRPC_ERROR_NONE); +} -/** Call data. Holds a pointer to SubchannelCall and the - associated machinery to create such a pointer. - Handles queueing of stream ops until a call object is ready, waiting - for initial metadata before trying to create a call object, - and handling cancellation gracefully. */ -struct call_data { - call_data(grpc_call_element* elem, const ChannelData& chand, - const grpc_call_element_args& args) - : deadline_state(elem, args.call_stack, args.call_combiner, - GPR_LIKELY(chand.deadline_checking_enabled()) - ? args.deadline - : GRPC_MILLIS_INF_FUTURE), - path(grpc_slice_ref_internal(args.path)), - call_start_time(args.start_time), - deadline(args.deadline), - arena(args.arena), - owning_call(args.call_stack), - call_combiner(args.call_combiner), - call_context(args.context), - pending_send_initial_metadata(false), - pending_send_message(false), - pending_send_trailing_metadata(false), - enable_retries(chand.enable_retries()), - retry_committed(false), - last_attempt_got_server_pushback(false) {} - - ~call_data() { - grpc_slice_unref_internal(path); - GRPC_ERROR_UNREF(cancel_error); - for (size_t i = 0; i < GPR_ARRAY_SIZE(pending_batches); ++i) { - GPR_ASSERT(pending_batches[i].batch == nullptr); +void CallData::StartTransportStreamOpBatch( + grpc_call_element* elem, grpc_transport_stream_op_batch* batch) { + GPR_TIMER_SCOPE("cc_start_transport_stream_op_batch", 0); + CallData* calld = static_cast(elem->call_data); + ChannelData* chand = static_cast(elem->channel_data); + if (GPR_LIKELY(chand->deadline_checking_enabled())) { + grpc_deadline_state_client_start_transport_stream_op_batch(elem, batch); + } + // If we've previously been cancelled, immediately fail any new batches. + if (GPR_UNLIKELY(calld->cancel_error_ != GRPC_ERROR_NONE)) { + if (grpc_client_channel_call_trace.enabled()) { + gpr_log(GPR_INFO, "chand=%p calld=%p: failing batch with error: %s", + chand, calld, grpc_error_string(calld->cancel_error_)); } + // Note: This will release the call combiner. + grpc_transport_stream_op_batch_finish_with_failure( + batch, GRPC_ERROR_REF(calld->cancel_error_), calld->call_combiner_); + return; } + // Handle cancellation. + if (GPR_UNLIKELY(batch->cancel_stream)) { + // Stash a copy of cancel_error in our call data, so that we can use + // it for subsequent operations. This ensures that if the call is + // cancelled before any batches are passed down (e.g., if the deadline + // is in the past when the call starts), we can return the right + // error to the caller when the first batch does get passed down. + GRPC_ERROR_UNREF(calld->cancel_error_); + calld->cancel_error_ = + GRPC_ERROR_REF(batch->payload->cancel_stream.cancel_error); + if (grpc_client_channel_call_trace.enabled()) { + gpr_log(GPR_INFO, "chand=%p calld=%p: recording cancel_error=%s", chand, + calld, grpc_error_string(calld->cancel_error_)); + } + // If we do not have a subchannel call (i.e., a pick has not yet + // been started), fail all pending batches. Otherwise, send the + // cancellation down to the subchannel call. + if (calld->subchannel_call_ == nullptr) { + // TODO(roth): If there is a pending retry callback, do we need to + // cancel it here? + calld->PendingBatchesFail(elem, GRPC_ERROR_REF(calld->cancel_error_), + NoYieldCallCombiner); + // Note: This will release the call combiner. + grpc_transport_stream_op_batch_finish_with_failure( + batch, GRPC_ERROR_REF(calld->cancel_error_), calld->call_combiner_); + } else { + // Note: This will release the call combiner. + calld->subchannel_call_->StartTransportStreamOpBatch(batch); + } + return; + } + // Add the batch to the pending list. + calld->PendingBatchesAdd(elem, batch); + // Check if we've already gotten a subchannel call. + // Note that once we have completed the pick, we do not need to enter + // the channel combiner, which is more efficient (especially for + // streaming calls). + if (calld->subchannel_call_ != nullptr) { + if (grpc_client_channel_call_trace.enabled()) { + gpr_log(GPR_INFO, + "chand=%p calld=%p: starting batch on subchannel_call=%p", chand, + calld, calld->subchannel_call_.get()); + } + calld->PendingBatchesResume(elem); + return; + } + // We do not yet have a subchannel call. + // For batches containing a send_initial_metadata op, enter the channel + // combiner to start a pick. + if (GPR_LIKELY(batch->send_initial_metadata)) { + if (grpc_client_channel_call_trace.enabled()) { + gpr_log(GPR_INFO, "chand=%p calld=%p: entering client_channel combiner", + chand, calld); + } + GRPC_CLOSURE_SCHED( + GRPC_CLOSURE_INIT( + &batch->handler_private.closure, StartPickLocked, elem, + grpc_combiner_scheduler(chand->data_plane_combiner())), + GRPC_ERROR_NONE); + } else { + // For all other batches, release the call combiner. + if (grpc_client_channel_call_trace.enabled()) { + gpr_log(GPR_INFO, + "chand=%p calld=%p: saved batch, yielding call combiner", chand, + calld); + } + GRPC_CALL_COMBINER_STOP(calld->call_combiner_, + "batch does not include send_initial_metadata"); + } +} - // State for handling deadlines. - // The code in deadline_filter.c requires this to be the first field. - // TODO(roth): This is slightly sub-optimal in that grpc_deadline_state - // and this struct both independently store pointers to the call stack - // and call combiner. If/when we have time, find a way to avoid this - // without breaking the grpc_deadline_state abstraction. - grpc_deadline_state deadline_state; - - grpc_slice path; // Request path. - gpr_timespec call_start_time; - grpc_millis deadline; - gpr_arena* arena; - grpc_call_stack* owning_call; - grpc_call_combiner* call_combiner; - grpc_call_context_element* call_context; - - grpc_core::RefCountedPtr retry_throttle_data; - grpc_core::RefCountedPtr method_params; - - grpc_core::RefCountedPtr subchannel_call; - - // Set when we get a cancel_stream op. - grpc_error* cancel_error = GRPC_ERROR_NONE; - - ChannelData::QueuedPick pick; - bool pick_queued = false; - bool service_config_applied = false; - grpc_core::QueuedPickCanceller* pick_canceller = nullptr; - grpc_closure pick_closure; - - grpc_polling_entity* pollent = nullptr; - - // Batches are added to this list when received from above. - // They are removed when we are done handling the batch (i.e., when - // either we have invoked all of the batch's callbacks or we have - // passed the batch down to the subchannel call and are not - // intercepting any of its callbacks). - pending_batch pending_batches[MAX_PENDING_BATCHES] = {}; - bool pending_send_initial_metadata : 1; - bool pending_send_message : 1; - bool pending_send_trailing_metadata : 1; - - // Retry state. - bool enable_retries : 1; - bool retry_committed : 1; - bool last_attempt_got_server_pushback : 1; - int num_attempts_completed = 0; - size_t bytes_buffered_for_retry = 0; - grpc_core::ManualConstructor retry_backoff; - grpc_timer retry_timer; - - // The number of pending retriable subchannel batches containing send ops. - // We hold a ref to the call stack while this is non-zero, since replay - // batches may not complete until after all callbacks have been returned - // to the surface, and we need to make sure that the call is not destroyed - // until all of these batches have completed. - // Note that we actually only need to track replay batches, but it's - // easier to track all batches with send ops. - int num_pending_retriable_subchannel_send_batches = 0; - - // Cached data for retrying send ops. - // send_initial_metadata - bool seen_send_initial_metadata = false; - grpc_linked_mdelem* send_initial_metadata_storage = nullptr; - grpc_metadata_batch send_initial_metadata; - uint32_t send_initial_metadata_flags; - gpr_atm* peer_string; - // send_message - // When we get a send_message op, we replace the original byte stream - // with a CachingByteStream that caches the slices to a local buffer for - // use in retries. - // Note: We inline the cache for the first 3 send_message ops and use - // dynamic allocation after that. This number was essentially picked - // at random; it could be changed in the future to tune performance. - grpc_core::InlinedVector send_messages; - // send_trailing_metadata - bool seen_send_trailing_metadata = false; - grpc_linked_mdelem* send_trailing_metadata_storage = nullptr; - grpc_metadata_batch send_trailing_metadata; -}; - -} // namespace - -// Forward declarations. -static void retry_commit(grpc_call_element* elem, - subchannel_call_retry_state* retry_state); -static void start_internal_recv_trailing_metadata(grpc_call_element* elem); -static void on_complete(void* arg, grpc_error* error); -static void start_retriable_subchannel_batches(void* arg, grpc_error* ignored); -static void remove_call_from_queued_picks_locked(grpc_call_element* elem); +void CallData::SetPollent(grpc_call_element* elem, + grpc_polling_entity* pollent) { + CallData* calld = static_cast(elem->call_data); + calld->pollent_ = pollent; +} // // send op data caching // -// Caches data for send ops so that it can be retried later, if not -// already cached. -static void maybe_cache_send_ops_for_batch(call_data* calld, - pending_batch* pending) { +void CallData::MaybeCacheSendOpsForBatch(PendingBatch* pending) { if (pending->send_ops_cached) return; pending->send_ops_cached = true; grpc_transport_stream_op_batch* batch = pending->batch; // Save a copy of metadata for send_initial_metadata ops. if (batch->send_initial_metadata) { - calld->seen_send_initial_metadata = true; - GPR_ASSERT(calld->send_initial_metadata_storage == nullptr); + seen_send_initial_metadata_ = true; + GPR_ASSERT(send_initial_metadata_storage_ == nullptr); grpc_metadata_batch* send_initial_metadata = batch->payload->send_initial_metadata.send_initial_metadata; - calld->send_initial_metadata_storage = (grpc_linked_mdelem*)gpr_arena_alloc( - calld->arena, - sizeof(grpc_linked_mdelem) * send_initial_metadata->list.count); - grpc_metadata_batch_copy(send_initial_metadata, - &calld->send_initial_metadata, - calld->send_initial_metadata_storage); - calld->send_initial_metadata_flags = + send_initial_metadata_storage_ = (grpc_linked_mdelem*)gpr_arena_alloc( + arena_, sizeof(grpc_linked_mdelem) * send_initial_metadata->list.count); + grpc_metadata_batch_copy(send_initial_metadata, &send_initial_metadata_, + send_initial_metadata_storage_); + send_initial_metadata_flags_ = batch->payload->send_initial_metadata.send_initial_metadata_flags; - calld->peer_string = batch->payload->send_initial_metadata.peer_string; + peer_string_ = batch->payload->send_initial_metadata.peer_string; } // Set up cache for send_message ops. if (batch->send_message) { - grpc_core::ByteStreamCache* cache = - static_cast( - gpr_arena_alloc(calld->arena, sizeof(grpc_core::ByteStreamCache))); - new (cache) grpc_core::ByteStreamCache( - std::move(batch->payload->send_message.send_message)); - calld->send_messages.push_back(cache); + ByteStreamCache* cache = static_cast( + gpr_arena_alloc(arena_, sizeof(ByteStreamCache))); + new (cache) + ByteStreamCache(std::move(batch->payload->send_message.send_message)); + send_messages_.push_back(cache); } // Save metadata batch for send_trailing_metadata ops. if (batch->send_trailing_metadata) { - calld->seen_send_trailing_metadata = true; - GPR_ASSERT(calld->send_trailing_metadata_storage == nullptr); + seen_send_trailing_metadata_ = true; + GPR_ASSERT(send_trailing_metadata_storage_ == nullptr); grpc_metadata_batch* send_trailing_metadata = batch->payload->send_trailing_metadata.send_trailing_metadata; - calld->send_trailing_metadata_storage = - (grpc_linked_mdelem*)gpr_arena_alloc( - calld->arena, - sizeof(grpc_linked_mdelem) * send_trailing_metadata->list.count); - grpc_metadata_batch_copy(send_trailing_metadata, - &calld->send_trailing_metadata, - calld->send_trailing_metadata_storage); + send_trailing_metadata_storage_ = (grpc_linked_mdelem*)gpr_arena_alloc( + arena_, + sizeof(grpc_linked_mdelem) * send_trailing_metadata->list.count); + grpc_metadata_batch_copy(send_trailing_metadata, &send_trailing_metadata_, + send_trailing_metadata_storage_); } } -// Frees cached send_initial_metadata. -static void free_cached_send_initial_metadata(ChannelData* chand, - call_data* calld) { +void CallData::FreeCachedSendInitialMetadata(ChannelData* chand) { if (grpc_client_channel_call_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: destroying calld->send_initial_metadata", chand, - calld); + this); } - grpc_metadata_batch_destroy(&calld->send_initial_metadata); + grpc_metadata_batch_destroy(&send_initial_metadata_); } -// Frees cached send_message at index idx. -static void free_cached_send_message(ChannelData* chand, call_data* calld, - size_t idx) { +void CallData::FreeCachedSendMessage(ChannelData* chand, size_t idx) { if (grpc_client_channel_call_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: destroying calld->send_messages[%" PRIuPTR "]", - chand, calld, idx); + chand, this, idx); } - calld->send_messages[idx]->Destroy(); + send_messages_[idx]->Destroy(); } -// Frees cached send_trailing_metadata. -static void free_cached_send_trailing_metadata(ChannelData* chand, - call_data* calld) { +void CallData::FreeCachedSendTrailingMetadata(ChannelData* chand) { if (grpc_client_channel_call_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: destroying calld->send_trailing_metadata", - chand, calld); + chand, this); } - grpc_metadata_batch_destroy(&calld->send_trailing_metadata); + grpc_metadata_batch_destroy(&send_trailing_metadata_); } -// Frees cached send ops that have already been completed after -// committing the call. -static void free_cached_send_op_data_after_commit( - grpc_call_element* elem, subchannel_call_retry_state* retry_state) { +void CallData::FreeCachedSendOpDataAfterCommit( + grpc_call_element* elem, SubchannelCallRetryState* retry_state) { ChannelData* chand = static_cast(elem->channel_data); - call_data* calld = static_cast(elem->call_data); if (retry_state->completed_send_initial_metadata) { - free_cached_send_initial_metadata(chand, calld); + FreeCachedSendInitialMetadata(chand); } for (size_t i = 0; i < retry_state->completed_send_message_count; ++i) { - free_cached_send_message(chand, calld, i); + FreeCachedSendMessage(chand, i); } if (retry_state->completed_send_trailing_metadata) { - free_cached_send_trailing_metadata(chand, calld); + FreeCachedSendTrailingMetadata(chand); } } -// Frees cached send ops that were completed by the completed batch in -// batch_data. Used when batches are completed after the call is committed. -static void free_cached_send_op_data_for_completed_batch( - grpc_call_element* elem, subchannel_batch_data* batch_data, - subchannel_call_retry_state* retry_state) { +void CallData::FreeCachedSendOpDataForCompletedBatch( + grpc_call_element* elem, SubchannelCallBatchData* batch_data, + SubchannelCallRetryState* retry_state) { ChannelData* chand = static_cast(elem->channel_data); - call_data* calld = static_cast(elem->call_data); if (batch_data->batch.send_initial_metadata) { - free_cached_send_initial_metadata(chand, calld); + FreeCachedSendInitialMetadata(chand); } if (batch_data->batch.send_message) { - free_cached_send_message(chand, calld, - retry_state->completed_send_message_count - 1); + FreeCachedSendMessage(chand, retry_state->completed_send_message_count - 1); } if (batch_data->batch.send_trailing_metadata) { - free_cached_send_trailing_metadata(chand, calld); + FreeCachedSendTrailingMetadata(chand); } } @@ -1294,7 +1573,7 @@ static void free_cached_send_op_data_for_completed_batch( // LB recv_trailing_metadata_ready handling // -void maybe_inject_recv_trailing_metadata_ready_for_lb( +void CallData::MaybeInjectRecvTrailingMetadataReadyForLoadBalancingPolicy( const LoadBalancingPolicy::PickArgs& pick, grpc_transport_stream_op_batch* batch) { if (pick.recv_trailing_metadata_ready != nullptr) { @@ -1313,8 +1592,7 @@ void maybe_inject_recv_trailing_metadata_ready_for_lb( // pending_batches management // -// Returns the index into calld->pending_batches to be used for batch. -static size_t get_batch_index(grpc_transport_stream_op_batch* batch) { +size_t CallData::GetBatchIndex(grpc_transport_stream_op_batch* batch) { // Note: It is important the send_initial_metadata be the first entry // here, since the code in pick_subchannel_locked() assumes it will be. if (batch->send_initial_metadata) return 0; @@ -1327,240 +1605,215 @@ static size_t get_batch_index(grpc_transport_stream_op_batch* batch) { } // This is called via the call combiner, so access to calld is synchronized. -static void pending_batches_add(grpc_call_element* elem, - grpc_transport_stream_op_batch* batch) { +void CallData::PendingBatchesAdd(grpc_call_element* elem, + grpc_transport_stream_op_batch* batch) { ChannelData* chand = static_cast(elem->channel_data); - call_data* calld = static_cast(elem->call_data); - const size_t idx = get_batch_index(batch); + const size_t idx = GetBatchIndex(batch); if (grpc_client_channel_call_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: adding pending batch at index %" PRIuPTR, chand, - calld, idx); + this, idx); } - pending_batch* pending = &calld->pending_batches[idx]; + PendingBatch* pending = &pending_batches_[idx]; GPR_ASSERT(pending->batch == nullptr); pending->batch = batch; pending->send_ops_cached = false; - if (calld->enable_retries) { + if (enable_retries_) { // Update state in calld about pending batches. // Also check if the batch takes us over the retry buffer limit. // Note: We don't check the size of trailing metadata here, because // gRPC clients do not send trailing metadata. if (batch->send_initial_metadata) { - calld->pending_send_initial_metadata = true; - calld->bytes_buffered_for_retry += grpc_metadata_batch_size( + pending_send_initial_metadata_ = true; + bytes_buffered_for_retry_ += grpc_metadata_batch_size( batch->payload->send_initial_metadata.send_initial_metadata); } if (batch->send_message) { - calld->pending_send_message = true; - calld->bytes_buffered_for_retry += + pending_send_message_ = true; + bytes_buffered_for_retry_ += batch->payload->send_message.send_message->length(); } if (batch->send_trailing_metadata) { - calld->pending_send_trailing_metadata = true; + pending_send_trailing_metadata_ = true; } - if (GPR_UNLIKELY(calld->bytes_buffered_for_retry > + if (GPR_UNLIKELY(bytes_buffered_for_retry_ > chand->per_rpc_retry_buffer_size())) { if (grpc_client_channel_call_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: exceeded retry buffer size, committing", - chand, calld); + chand, this); } - subchannel_call_retry_state* retry_state = - calld->subchannel_call == nullptr - ? nullptr - : static_cast( - - calld->subchannel_call->GetParentData()); - retry_commit(elem, retry_state); + SubchannelCallRetryState* retry_state = + subchannel_call_ == nullptr ? nullptr + : static_cast( + subchannel_call_->GetParentData()); + RetryCommit(elem, retry_state); // If we are not going to retry and have not yet started, pretend // retries are disabled so that we don't bother with retry overhead. - if (calld->num_attempts_completed == 0) { + if (num_attempts_completed_ == 0) { if (grpc_client_channel_call_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: disabling retries before first attempt", - chand, calld); + chand, this); } - calld->enable_retries = false; + enable_retries_ = false; } } } } -static void pending_batch_clear(call_data* calld, pending_batch* pending) { - if (calld->enable_retries) { +void CallData::PendingBatchClear(PendingBatch* pending) { + if (enable_retries_) { if (pending->batch->send_initial_metadata) { - calld->pending_send_initial_metadata = false; + pending_send_initial_metadata_ = false; } if (pending->batch->send_message) { - calld->pending_send_message = false; + pending_send_message_ = false; } if (pending->batch->send_trailing_metadata) { - calld->pending_send_trailing_metadata = false; + pending_send_trailing_metadata_ = false; } } pending->batch = nullptr; } +void CallData::MaybeClearPendingBatch(grpc_call_element* elem, + PendingBatch* pending) { + ChannelData* chand = static_cast(elem->channel_data); + grpc_transport_stream_op_batch* batch = pending->batch; + // We clear the pending batch if all of its callbacks have been + // scheduled and reset to nullptr. + if (batch->on_complete == nullptr && + (!batch->recv_initial_metadata || + batch->payload->recv_initial_metadata.recv_initial_metadata_ready == + nullptr) && + (!batch->recv_message || + batch->payload->recv_message.recv_message_ready == nullptr) && + (!batch->recv_trailing_metadata || + batch->payload->recv_trailing_metadata.recv_trailing_metadata_ready == + nullptr)) { + if (grpc_client_channel_call_trace.enabled()) { + gpr_log(GPR_INFO, "chand=%p calld=%p: clearing pending batch", chand, + this); + } + PendingBatchClear(pending); + } +} + // This is called via the call combiner, so access to calld is synchronized. -static void fail_pending_batch_in_call_combiner(void* arg, grpc_error* error) { +void CallData::FailPendingBatchInCallCombiner(void* arg, grpc_error* error) { grpc_transport_stream_op_batch* batch = static_cast(arg); - call_data* calld = static_cast(batch->handler_private.extra_arg); + CallData* calld = static_cast(batch->handler_private.extra_arg); // Note: This will release the call combiner. grpc_transport_stream_op_batch_finish_with_failure( - batch, GRPC_ERROR_REF(error), calld->call_combiner); + batch, GRPC_ERROR_REF(error), calld->call_combiner_); } // This is called via the call combiner, so access to calld is synchronized. -// If yield_call_combiner_predicate returns true, assumes responsibility for -// yielding the call combiner. -typedef bool (*YieldCallCombinerPredicate)( - const grpc_core::CallCombinerClosureList& closures); -static bool yield_call_combiner( - const grpc_core::CallCombinerClosureList& closures) { - return true; -} -static bool no_yield_call_combiner( - const grpc_core::CallCombinerClosureList& closures) { - return false; -} -static bool yield_call_combiner_if_pending_batches_found( - const grpc_core::CallCombinerClosureList& closures) { - return closures.size() > 0; -} -static void pending_batches_fail( +void CallData::PendingBatchesFail( grpc_call_element* elem, grpc_error* error, YieldCallCombinerPredicate yield_call_combiner_predicate) { GPR_ASSERT(error != GRPC_ERROR_NONE); - call_data* calld = static_cast(elem->call_data); if (grpc_client_channel_call_trace.enabled()) { size_t num_batches = 0; - for (size_t i = 0; i < GPR_ARRAY_SIZE(calld->pending_batches); ++i) { - if (calld->pending_batches[i].batch != nullptr) ++num_batches; + for (size_t i = 0; i < GPR_ARRAY_SIZE(pending_batches_); ++i) { + if (pending_batches_[i].batch != nullptr) ++num_batches; } gpr_log(GPR_INFO, "chand=%p calld=%p: failing %" PRIuPTR " pending batches: %s", - elem->channel_data, calld, num_batches, grpc_error_string(error)); + elem->channel_data, this, num_batches, grpc_error_string(error)); } - grpc_core::CallCombinerClosureList closures; - for (size_t i = 0; i < GPR_ARRAY_SIZE(calld->pending_batches); ++i) { - pending_batch* pending = &calld->pending_batches[i]; + CallCombinerClosureList closures; + for (size_t i = 0; i < GPR_ARRAY_SIZE(pending_batches_); ++i) { + PendingBatch* pending = &pending_batches_[i]; grpc_transport_stream_op_batch* batch = pending->batch; if (batch != nullptr) { if (batch->recv_trailing_metadata) { - maybe_inject_recv_trailing_metadata_ready_for_lb(calld->pick.pick, - batch); + MaybeInjectRecvTrailingMetadataReadyForLoadBalancingPolicy(pick_.pick, + batch); } - batch->handler_private.extra_arg = calld; + batch->handler_private.extra_arg = this; GRPC_CLOSURE_INIT(&batch->handler_private.closure, - fail_pending_batch_in_call_combiner, batch, + FailPendingBatchInCallCombiner, batch, grpc_schedule_on_exec_ctx); closures.Add(&batch->handler_private.closure, GRPC_ERROR_REF(error), - "pending_batches_fail"); - pending_batch_clear(calld, pending); + "PendingBatchesFail"); + PendingBatchClear(pending); } } if (yield_call_combiner_predicate(closures)) { - closures.RunClosures(calld->call_combiner); + closures.RunClosures(call_combiner_); } else { - closures.RunClosuresWithoutYielding(calld->call_combiner); + closures.RunClosuresWithoutYielding(call_combiner_); } GRPC_ERROR_UNREF(error); } // This is called via the call combiner, so access to calld is synchronized. -static void resume_pending_batch_in_call_combiner(void* arg, - grpc_error* ignored) { +void CallData::ResumePendingBatchInCallCombiner(void* arg, + grpc_error* ignored) { grpc_transport_stream_op_batch* batch = static_cast(arg); - grpc_core::SubchannelCall* subchannel_call = - static_cast(batch->handler_private.extra_arg); + SubchannelCall* subchannel_call = + static_cast(batch->handler_private.extra_arg); // Note: This will release the call combiner. subchannel_call->StartTransportStreamOpBatch(batch); } // This is called via the call combiner, so access to calld is synchronized. -static void pending_batches_resume(grpc_call_element* elem) { +void CallData::PendingBatchesResume(grpc_call_element* elem) { ChannelData* chand = static_cast(elem->channel_data); - call_data* calld = static_cast(elem->call_data); - if (calld->enable_retries) { - start_retriable_subchannel_batches(elem, GRPC_ERROR_NONE); + if (enable_retries_) { + StartRetriableSubchannelBatches(elem, GRPC_ERROR_NONE); return; } // Retries not enabled; send down batches as-is. if (grpc_client_channel_call_trace.enabled()) { size_t num_batches = 0; - for (size_t i = 0; i < GPR_ARRAY_SIZE(calld->pending_batches); ++i) { - if (calld->pending_batches[i].batch != nullptr) ++num_batches; + for (size_t i = 0; i < GPR_ARRAY_SIZE(pending_batches_); ++i) { + if (pending_batches_[i].batch != nullptr) ++num_batches; } gpr_log(GPR_INFO, "chand=%p calld=%p: starting %" PRIuPTR " pending batches on subchannel_call=%p", - chand, calld, num_batches, calld->subchannel_call.get()); + chand, this, num_batches, subchannel_call_.get()); } - grpc_core::CallCombinerClosureList closures; - for (size_t i = 0; i < GPR_ARRAY_SIZE(calld->pending_batches); ++i) { - pending_batch* pending = &calld->pending_batches[i]; + CallCombinerClosureList closures; + for (size_t i = 0; i < GPR_ARRAY_SIZE(pending_batches_); ++i) { + PendingBatch* pending = &pending_batches_[i]; grpc_transport_stream_op_batch* batch = pending->batch; if (batch != nullptr) { if (batch->recv_trailing_metadata) { - maybe_inject_recv_trailing_metadata_ready_for_lb(calld->pick.pick, - batch); + MaybeInjectRecvTrailingMetadataReadyForLoadBalancingPolicy(pick_.pick, + batch); } - batch->handler_private.extra_arg = calld->subchannel_call.get(); + batch->handler_private.extra_arg = subchannel_call_.get(); GRPC_CLOSURE_INIT(&batch->handler_private.closure, - resume_pending_batch_in_call_combiner, batch, + ResumePendingBatchInCallCombiner, batch, grpc_schedule_on_exec_ctx); closures.Add(&batch->handler_private.closure, GRPC_ERROR_NONE, - "pending_batches_resume"); - pending_batch_clear(calld, pending); + "PendingBatchesResume"); + PendingBatchClear(pending); } } // Note: This will release the call combiner. - closures.RunClosures(calld->call_combiner); + closures.RunClosures(call_combiner_); } -static void maybe_clear_pending_batch(grpc_call_element* elem, - pending_batch* pending) { - ChannelData* chand = static_cast(elem->channel_data); - call_data* calld = static_cast(elem->call_data); - grpc_transport_stream_op_batch* batch = pending->batch; - // We clear the pending batch if all of its callbacks have been - // scheduled and reset to nullptr. - if (batch->on_complete == nullptr && - (!batch->recv_initial_metadata || - batch->payload->recv_initial_metadata.recv_initial_metadata_ready == - nullptr) && - (!batch->recv_message || - batch->payload->recv_message.recv_message_ready == nullptr) && - (!batch->recv_trailing_metadata || - batch->payload->recv_trailing_metadata.recv_trailing_metadata_ready == - nullptr)) { - if (grpc_client_channel_call_trace.enabled()) { - gpr_log(GPR_INFO, "chand=%p calld=%p: clearing pending batch", chand, - calld); - } - pending_batch_clear(calld, pending); - } -} - -// Returns a pointer to the first pending batch for which predicate(batch) -// returns true, or null if not found. template -static pending_batch* pending_batch_find(grpc_call_element* elem, - const char* log_message, - Predicate predicate) { +CallData::PendingBatch* CallData::PendingBatchFind(grpc_call_element* elem, + const char* log_message, + Predicate predicate) { ChannelData* chand = static_cast(elem->channel_data); - call_data* calld = static_cast(elem->call_data); - for (size_t i = 0; i < GPR_ARRAY_SIZE(calld->pending_batches); ++i) { - pending_batch* pending = &calld->pending_batches[i]; + for (size_t i = 0; i < GPR_ARRAY_SIZE(pending_batches_); ++i) { + PendingBatch* pending = &pending_batches_[i]; grpc_transport_stream_op_batch* batch = pending->batch; if (batch != nullptr && predicate(batch)) { if (grpc_client_channel_call_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: %s pending batch at index %" PRIuPTR, chand, - calld, log_message, i); + this, log_message, i); } return pending; } @@ -1572,99 +1825,92 @@ static pending_batch* pending_batch_find(grpc_call_element* elem, // retry code // -// Commits the call so that no further retry attempts will be performed. -static void retry_commit(grpc_call_element* elem, - subchannel_call_retry_state* retry_state) { +void CallData::RetryCommit(grpc_call_element* elem, + SubchannelCallRetryState* retry_state) { ChannelData* chand = static_cast(elem->channel_data); - call_data* calld = static_cast(elem->call_data); - if (calld->retry_committed) return; - calld->retry_committed = true; + if (retry_committed_) return; + retry_committed_ = true; if (grpc_client_channel_call_trace.enabled()) { - gpr_log(GPR_INFO, "chand=%p calld=%p: committing retries", chand, calld); + gpr_log(GPR_INFO, "chand=%p calld=%p: committing retries", chand, this); } if (retry_state != nullptr) { - free_cached_send_op_data_after_commit(elem, retry_state); + FreeCachedSendOpDataAfterCommit(elem, retry_state); } } -// Starts a retry after appropriate back-off. -static void do_retry(grpc_call_element* elem, - subchannel_call_retry_state* retry_state, - grpc_millis server_pushback_ms) { +void CallData::DoRetry(grpc_call_element* elem, + SubchannelCallRetryState* retry_state, + grpc_millis server_pushback_ms) { ChannelData* chand = static_cast(elem->channel_data); - call_data* calld = static_cast(elem->call_data); - GPR_ASSERT(calld->method_params != nullptr); + GPR_ASSERT(method_params_ != nullptr); const ClientChannelMethodParams::RetryPolicy* retry_policy = - calld->method_params->retry_policy(); + method_params_->retry_policy(); GPR_ASSERT(retry_policy != nullptr); // Reset subchannel call and connected subchannel. - calld->subchannel_call.reset(); - calld->pick.pick.connected_subchannel.reset(); + subchannel_call_.reset(); + pick_.pick.connected_subchannel.reset(); // Compute backoff delay. grpc_millis next_attempt_time; if (server_pushback_ms >= 0) { next_attempt_time = grpc_core::ExecCtx::Get()->Now() + server_pushback_ms; - calld->last_attempt_got_server_pushback = true; + last_attempt_got_server_pushback_ = true; } else { - if (calld->num_attempts_completed == 1 || - calld->last_attempt_got_server_pushback) { - calld->retry_backoff.Init( - grpc_core::BackOff::Options() + if (num_attempts_completed_ == 1 || last_attempt_got_server_pushback_) { + retry_backoff_.Init( + BackOff::Options() .set_initial_backoff(retry_policy->initial_backoff) .set_multiplier(retry_policy->backoff_multiplier) .set_jitter(RETRY_BACKOFF_JITTER) .set_max_backoff(retry_policy->max_backoff)); - calld->last_attempt_got_server_pushback = false; + last_attempt_got_server_pushback_ = false; } - next_attempt_time = calld->retry_backoff->NextAttemptTime(); + next_attempt_time = retry_backoff_->NextAttemptTime(); } if (grpc_client_channel_call_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: retrying failed call in %" PRId64 " ms", chand, - calld, next_attempt_time - grpc_core::ExecCtx::Get()->Now()); + this, next_attempt_time - grpc_core::ExecCtx::Get()->Now()); } // Schedule retry after computed delay. - GRPC_CLOSURE_INIT(&calld->pick_closure, start_pick_locked, elem, + GRPC_CLOSURE_INIT(&pick_closure_, StartPickLocked, elem, grpc_combiner_scheduler(chand->data_plane_combiner())); - grpc_timer_init(&calld->retry_timer, next_attempt_time, &calld->pick_closure); + grpc_timer_init(&retry_timer_, next_attempt_time, &pick_closure_); // Update bookkeeping. if (retry_state != nullptr) retry_state->retry_dispatched = true; } -// Returns true if the call is being retried. -static bool maybe_retry(grpc_call_element* elem, - subchannel_batch_data* batch_data, - grpc_status_code status, - grpc_mdelem* server_pushback_md) { +bool CallData::MaybeRetry(grpc_call_element* elem, + SubchannelCallBatchData* batch_data, + grpc_status_code status, + grpc_mdelem* server_pushback_md) { ChannelData* chand = static_cast(elem->channel_data); - call_data* calld = static_cast(elem->call_data); // Get retry policy. - if (calld->method_params == nullptr) return false; + if (method_params_ == nullptr) return false; const ClientChannelMethodParams::RetryPolicy* retry_policy = - calld->method_params->retry_policy(); + method_params_->retry_policy(); if (retry_policy == nullptr) return false; // If we've already dispatched a retry from this call, return true. // This catches the case where the batch has multiple callbacks // (i.e., it includes either recv_message or recv_initial_metadata). - subchannel_call_retry_state* retry_state = nullptr; + SubchannelCallRetryState* retry_state = nullptr; if (batch_data != nullptr) { - retry_state = static_cast( + retry_state = static_cast( batch_data->subchannel_call->GetParentData()); if (retry_state->retry_dispatched) { if (grpc_client_channel_call_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: retry already dispatched", chand, - calld); + this); } return true; } } // Check status. if (GPR_LIKELY(status == GRPC_STATUS_OK)) { - if (calld->retry_throttle_data != nullptr) { - calld->retry_throttle_data->RecordSuccess(); + if (retry_throttle_data_ != nullptr) { + retry_throttle_data_->RecordSuccess(); } if (grpc_client_channel_call_trace.enabled()) { - gpr_log(GPR_INFO, "chand=%p calld=%p: call succeeded", chand, calld); + gpr_log(GPR_INFO, "chand=%p calld=%p: call succeeded", chand, this); } return false; } @@ -1673,7 +1919,7 @@ static bool maybe_retry(grpc_call_element* elem, if (grpc_client_channel_call_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: status %s not configured as retryable", chand, - calld, grpc_status_code_to_string(status)); + this, grpc_status_code_to_string(status)); } return false; } @@ -1684,36 +1930,36 @@ static bool maybe_retry(grpc_call_element* elem, // things like failures due to malformed requests (INVALID_ARGUMENT). // Conversely, it's important for this to come before the remaining // checks, so that we don't fail to record failures due to other factors. - if (calld->retry_throttle_data != nullptr && - !calld->retry_throttle_data->RecordFailure()) { + if (retry_throttle_data_ != nullptr && + !retry_throttle_data_->RecordFailure()) { if (grpc_client_channel_call_trace.enabled()) { - gpr_log(GPR_INFO, "chand=%p calld=%p: retries throttled", chand, calld); + gpr_log(GPR_INFO, "chand=%p calld=%p: retries throttled", chand, this); } return false; } // Check whether the call is committed. - if (calld->retry_committed) { + if (retry_committed_) { if (grpc_client_channel_call_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: retries already committed", chand, - calld); + this); } return false; } // Check whether we have retries remaining. - ++calld->num_attempts_completed; - if (calld->num_attempts_completed >= retry_policy->max_attempts) { + ++num_attempts_completed_; + if (num_attempts_completed_ >= retry_policy->max_attempts) { if (grpc_client_channel_call_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: exceeded %d retry attempts", chand, - calld, retry_policy->max_attempts); + this, retry_policy->max_attempts); } return false; } // If the call was cancelled from the surface, don't retry. - if (calld->cancel_error != GRPC_ERROR_NONE) { + if (cancel_error_ != GRPC_ERROR_NONE) { if (grpc_client_channel_call_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: call cancelled from surface, not retrying", - chand, calld); + chand, this); } return false; } @@ -1726,48 +1972,54 @@ static bool maybe_retry(grpc_call_element* elem, if (grpc_client_channel_call_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: not retrying due to server push-back", - chand, calld); + chand, this); } return false; } else { if (grpc_client_channel_call_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: server push-back: retry in %u ms", - chand, calld, ms); + chand, this, ms); } server_pushback_ms = (grpc_millis)ms; } } - do_retry(elem, retry_state, server_pushback_ms); + DoRetry(elem, retry_state, server_pushback_ms); return true; } // -// subchannel_batch_data +// CallData::SubchannelCallBatchData // -namespace { +CallData::SubchannelCallBatchData* CallData::SubchannelCallBatchData::Create( + grpc_call_element* elem, int refcount, bool set_on_complete) { + CallData* calld = static_cast(elem->call_data); + SubchannelCallBatchData* batch_data = + new (gpr_arena_alloc(calld->arena_, sizeof(*batch_data))) + SubchannelCallBatchData(elem, calld, refcount, set_on_complete); + return batch_data; +} -subchannel_batch_data::subchannel_batch_data(grpc_call_element* elem, - call_data* calld, int refcount, - bool set_on_complete) - : elem(elem), subchannel_call(calld->subchannel_call) { - subchannel_call_retry_state* retry_state = - static_cast( - calld->subchannel_call->GetParentData()); +CallData::SubchannelCallBatchData::SubchannelCallBatchData( + grpc_call_element* elem, CallData* calld, int refcount, + bool set_on_complete) + : elem(elem), subchannel_call(calld->subchannel_call_) { + SubchannelCallRetryState* retry_state = + static_cast( + calld->subchannel_call_->GetParentData()); batch.payload = &retry_state->batch_payload; gpr_ref_init(&refs, refcount); if (set_on_complete) { - GRPC_CLOSURE_INIT(&on_complete, ::on_complete, this, + GRPC_CLOSURE_INIT(&on_complete, CallData::OnComplete, this, grpc_schedule_on_exec_ctx); batch.on_complete = &on_complete; } - GRPC_CALL_STACK_REF(calld->owning_call, "batch_data"); + GRPC_CALL_STACK_REF(calld->owning_call_, "batch_data"); } -void subchannel_batch_data::destroy() { - subchannel_call_retry_state* retry_state = - static_cast( - subchannel_call->GetParentData()); +void CallData::SubchannelCallBatchData::Destroy() { + SubchannelCallRetryState* retry_state = + static_cast(subchannel_call->GetParentData()); if (batch.send_initial_metadata) { grpc_metadata_batch_destroy(&retry_state->send_initial_metadata); } @@ -1781,42 +2033,20 @@ void subchannel_batch_data::destroy() { grpc_metadata_batch_destroy(&retry_state->recv_trailing_metadata); } subchannel_call.reset(); - call_data* calld = static_cast(elem->call_data); - GRPC_CALL_STACK_UNREF(calld->owning_call, "batch_data"); -} - -} // namespace - -// Creates a subchannel_batch_data object on the call's arena with the -// specified refcount. If set_on_complete is true, the batch's -// on_complete callback will be set to point to on_complete(); -// otherwise, the batch's on_complete callback will be null. -static subchannel_batch_data* batch_data_create(grpc_call_element* elem, - int refcount, - bool set_on_complete) { - call_data* calld = static_cast(elem->call_data); - subchannel_batch_data* batch_data = - new (gpr_arena_alloc(calld->arena, sizeof(*batch_data))) - subchannel_batch_data(elem, calld, refcount, set_on_complete); - return batch_data; -} - -static void batch_data_unref(subchannel_batch_data* batch_data) { - if (gpr_unref(&batch_data->refs)) { - batch_data->destroy(); - } + CallData* calld = static_cast(elem->call_data); + GRPC_CALL_STACK_UNREF(calld->owning_call_, "batch_data"); } // // recv_initial_metadata callback handling // -// Invokes recv_initial_metadata_ready for a subchannel batch. -static void invoke_recv_initial_metadata_callback(void* arg, - grpc_error* error) { - subchannel_batch_data* batch_data = static_cast(arg); +void CallData::InvokeRecvInitialMetadataCallback(void* arg, grpc_error* error) { + SubchannelCallBatchData* batch_data = + static_cast(arg); + CallData* calld = static_cast(batch_data->elem->call_data); // Find pending batch. - pending_batch* pending = pending_batch_find( + PendingBatch* pending = calld->PendingBatchFind( batch_data->elem, "invoking recv_initial_metadata_ready for", [](grpc_transport_stream_op_batch* batch) { return batch->recv_initial_metadata && @@ -1825,8 +2055,8 @@ static void invoke_recv_initial_metadata_callback(void* arg, }); GPR_ASSERT(pending != nullptr); // Return metadata. - subchannel_call_retry_state* retry_state = - static_cast( + SubchannelCallRetryState* retry_state = + static_cast( batch_data->subchannel_call->GetParentData()); grpc_metadata_batch_move( &retry_state->recv_initial_metadata, @@ -1839,33 +2069,32 @@ static void invoke_recv_initial_metadata_callback(void* arg, .recv_initial_metadata_ready; pending->batch->payload->recv_initial_metadata.recv_initial_metadata_ready = nullptr; - maybe_clear_pending_batch(batch_data->elem, pending); - batch_data_unref(batch_data); + calld->MaybeClearPendingBatch(batch_data->elem, pending); + batch_data->Unref(); // Invoke callback. GRPC_CLOSURE_RUN(recv_initial_metadata_ready, GRPC_ERROR_REF(error)); } -// Intercepts recv_initial_metadata_ready callback for retries. -// Commits the call and returns the initial metadata up the stack. -static void recv_initial_metadata_ready(void* arg, grpc_error* error) { - subchannel_batch_data* batch_data = static_cast(arg); +void CallData::RecvInitialMetadataReady(void* arg, grpc_error* error) { + SubchannelCallBatchData* batch_data = + static_cast(arg); grpc_call_element* elem = batch_data->elem; ChannelData* chand = static_cast(elem->channel_data); - call_data* calld = static_cast(elem->call_data); + CallData* calld = static_cast(elem->call_data); if (grpc_client_channel_call_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: got recv_initial_metadata_ready, error=%s", chand, calld, grpc_error_string(error)); } - subchannel_call_retry_state* retry_state = - static_cast( + SubchannelCallRetryState* retry_state = + static_cast( batch_data->subchannel_call->GetParentData()); retry_state->completed_recv_initial_metadata = true; // If a retry was already dispatched, then we're not going to use the // result of this recv_initial_metadata op, so do nothing. if (retry_state->retry_dispatched) { GRPC_CALL_COMBINER_STOP( - calld->call_combiner, + calld->call_combiner_, "recv_initial_metadata_ready after retry dispatched"); return; } @@ -1887,30 +2116,31 @@ static void recv_initial_metadata_ready(void* arg, grpc_error* error) { if (!retry_state->started_recv_trailing_metadata) { // recv_trailing_metadata not yet started by application; start it // ourselves to get status. - start_internal_recv_trailing_metadata(elem); + calld->StartInternalRecvTrailingMetadata(elem); } else { GRPC_CALL_COMBINER_STOP( - calld->call_combiner, + calld->call_combiner_, "recv_initial_metadata_ready trailers-only or error"); } return; } // Received valid initial metadata, so commit the call. - retry_commit(elem, retry_state); + calld->RetryCommit(elem, retry_state); // Invoke the callback to return the result to the surface. // Manually invoking a callback function; it does not take ownership of error. - invoke_recv_initial_metadata_callback(batch_data, error); + calld->InvokeRecvInitialMetadataCallback(batch_data, error); } // // recv_message callback handling // -// Invokes recv_message_ready for a subchannel batch. -static void invoke_recv_message_callback(void* arg, grpc_error* error) { - subchannel_batch_data* batch_data = static_cast(arg); +void CallData::InvokeRecvMessageCallback(void* arg, grpc_error* error) { + SubchannelCallBatchData* batch_data = + static_cast(arg); + CallData* calld = static_cast(batch_data->elem->call_data); // Find pending op. - pending_batch* pending = pending_batch_find( + PendingBatch* pending = calld->PendingBatchFind( batch_data->elem, "invoking recv_message_ready for", [](grpc_transport_stream_op_batch* batch) { return batch->recv_message && @@ -1918,8 +2148,8 @@ static void invoke_recv_message_callback(void* arg, grpc_error* error) { }); GPR_ASSERT(pending != nullptr); // Return payload. - subchannel_call_retry_state* retry_state = - static_cast( + SubchannelCallRetryState* retry_state = + static_cast( batch_data->subchannel_call->GetParentData()); *pending->batch->payload->recv_message.recv_message = std::move(retry_state->recv_message); @@ -1929,31 +2159,30 @@ static void invoke_recv_message_callback(void* arg, grpc_error* error) { grpc_closure* recv_message_ready = pending->batch->payload->recv_message.recv_message_ready; pending->batch->payload->recv_message.recv_message_ready = nullptr; - maybe_clear_pending_batch(batch_data->elem, pending); - batch_data_unref(batch_data); + calld->MaybeClearPendingBatch(batch_data->elem, pending); + batch_data->Unref(); // Invoke callback. GRPC_CLOSURE_RUN(recv_message_ready, GRPC_ERROR_REF(error)); } -// Intercepts recv_message_ready callback for retries. -// Commits the call and returns the message up the stack. -static void recv_message_ready(void* arg, grpc_error* error) { - subchannel_batch_data* batch_data = static_cast(arg); +void CallData::RecvMessageReady(void* arg, grpc_error* error) { + SubchannelCallBatchData* batch_data = + static_cast(arg); grpc_call_element* elem = batch_data->elem; ChannelData* chand = static_cast(elem->channel_data); - call_data* calld = static_cast(elem->call_data); + CallData* calld = static_cast(elem->call_data); if (grpc_client_channel_call_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: got recv_message_ready, error=%s", chand, calld, grpc_error_string(error)); } - subchannel_call_retry_state* retry_state = - static_cast( + SubchannelCallRetryState* retry_state = + static_cast( batch_data->subchannel_call->GetParentData()); ++retry_state->completed_recv_message_count; // If a retry was already dispatched, then we're not going to use the // result of this recv_message op, so do nothing. if (retry_state->retry_dispatched) { - GRPC_CALL_COMBINER_STOP(calld->call_combiner, + GRPC_CALL_COMBINER_STOP(calld->call_combiner_, "recv_message_ready after retry dispatched"); return; } @@ -1975,33 +2204,29 @@ static void recv_message_ready(void* arg, grpc_error* error) { if (!retry_state->started_recv_trailing_metadata) { // recv_trailing_metadata not yet started by application; start it // ourselves to get status. - start_internal_recv_trailing_metadata(elem); + calld->StartInternalRecvTrailingMetadata(elem); } else { - GRPC_CALL_COMBINER_STOP(calld->call_combiner, "recv_message_ready null"); + GRPC_CALL_COMBINER_STOP(calld->call_combiner_, "recv_message_ready null"); } return; } // Received a valid message, so commit the call. - retry_commit(elem, retry_state); + calld->RetryCommit(elem, retry_state); // Invoke the callback to return the result to the surface. // Manually invoking a callback function; it does not take ownership of error. - invoke_recv_message_callback(batch_data, error); + calld->InvokeRecvMessageCallback(batch_data, error); } // // recv_trailing_metadata handling // -// Sets *status and *server_pushback_md based on md_batch and error. -// Only sets *server_pushback_md if server_pushback_md != nullptr. -static void get_call_status(grpc_call_element* elem, - grpc_metadata_batch* md_batch, grpc_error* error, - grpc_status_code* status, - grpc_mdelem** server_pushback_md) { - call_data* calld = static_cast(elem->call_data); +void CallData::GetCallStatus(grpc_call_element* elem, + grpc_metadata_batch* md_batch, grpc_error* error, + grpc_status_code* status, + grpc_mdelem** server_pushback_md) { if (error != GRPC_ERROR_NONE) { - grpc_error_get_status(error, calld->deadline, status, nullptr, nullptr, - nullptr); + grpc_error_get_status(error, deadline_, status, nullptr, nullptr, nullptr); } else { GPR_ASSERT(md_batch->idx.named.grpc_status != nullptr); *status = @@ -2014,12 +2239,11 @@ static void get_call_status(grpc_call_element* elem, GRPC_ERROR_UNREF(error); } -// Adds recv_trailing_metadata_ready closure to closures. -static void add_closure_for_recv_trailing_metadata_ready( - grpc_call_element* elem, subchannel_batch_data* batch_data, - grpc_error* error, grpc_core::CallCombinerClosureList* closures) { +void CallData::AddClosureForRecvTrailingMetadataReady( + grpc_call_element* elem, SubchannelCallBatchData* batch_data, + grpc_error* error, CallCombinerClosureList* closures) { // Find pending batch. - pending_batch* pending = pending_batch_find( + PendingBatch* pending = PendingBatchFind( elem, "invoking recv_trailing_metadata for", [](grpc_transport_stream_op_batch* batch) { return batch->recv_trailing_metadata && @@ -2027,15 +2251,14 @@ static void add_closure_for_recv_trailing_metadata_ready( .recv_trailing_metadata_ready != nullptr; }); // If we generated the recv_trailing_metadata op internally via - // start_internal_recv_trailing_metadata(), then there will be no - // pending batch. + // StartInternalRecvTrailingMetadata(), then there will be no pending batch. if (pending == nullptr) { GRPC_ERROR_UNREF(error); return; } // Return metadata. - subchannel_call_retry_state* retry_state = - static_cast( + SubchannelCallRetryState* retry_state = + static_cast( batch_data->subchannel_call->GetParentData()); grpc_metadata_batch_move( &retry_state->recv_trailing_metadata, @@ -2047,20 +2270,18 @@ static void add_closure_for_recv_trailing_metadata_ready( // Update bookkeeping. pending->batch->payload->recv_trailing_metadata.recv_trailing_metadata_ready = nullptr; - maybe_clear_pending_batch(elem, pending); + MaybeClearPendingBatch(elem, pending); } -// Adds any necessary closures for deferred recv_initial_metadata and -// recv_message callbacks to closures. -static void add_closures_for_deferred_recv_callbacks( - subchannel_batch_data* batch_data, subchannel_call_retry_state* retry_state, - grpc_core::CallCombinerClosureList* closures) { +void CallData::AddClosuresForDeferredRecvCallbacks( + SubchannelCallBatchData* batch_data, SubchannelCallRetryState* retry_state, + CallCombinerClosureList* closures) { if (batch_data->batch.recv_trailing_metadata) { // Add closure for deferred recv_initial_metadata_ready. if (GPR_UNLIKELY(retry_state->recv_initial_metadata_ready_deferred_batch != nullptr)) { GRPC_CLOSURE_INIT(&retry_state->recv_initial_metadata_ready, - invoke_recv_initial_metadata_callback, + InvokeRecvInitialMetadataCallback, retry_state->recv_initial_metadata_ready_deferred_batch, grpc_schedule_on_exec_ctx); closures->Add(&retry_state->recv_initial_metadata_ready, @@ -2072,7 +2293,7 @@ static void add_closures_for_deferred_recv_callbacks( if (GPR_UNLIKELY(retry_state->recv_message_ready_deferred_batch != nullptr)) { GRPC_CLOSURE_INIT(&retry_state->recv_message_ready, - invoke_recv_message_callback, + InvokeRecvMessageCallback, retry_state->recv_message_ready_deferred_batch, grpc_schedule_on_exec_ctx); closures->Add(&retry_state->recv_message_ready, @@ -2083,11 +2304,8 @@ static void add_closures_for_deferred_recv_callbacks( } } -// Returns true if any op in the batch was not yet started. -// Only looks at send ops, since recv ops are always started immediately. -static bool pending_batch_is_unstarted( - pending_batch* pending, call_data* calld, - subchannel_call_retry_state* retry_state) { +bool CallData::PendingBatchIsUnstarted(PendingBatch* pending, + SubchannelCallRetryState* retry_state) { if (pending->batch == nullptr || pending->batch->on_complete == nullptr) { return false; } @@ -2096,7 +2314,7 @@ static bool pending_batch_is_unstarted( return true; } if (pending->batch->send_message && - retry_state->started_send_message_count < calld->send_messages.size()) { + retry_state->started_send_message_count < send_messages_.size()) { return true; } if (pending->batch->send_trailing_metadata && @@ -2106,72 +2324,66 @@ static bool pending_batch_is_unstarted( return false; } -// For any pending batch containing an op that has not yet been started, -// adds the pending batch's completion closures to closures. -static void add_closures_to_fail_unstarted_pending_batches( - grpc_call_element* elem, subchannel_call_retry_state* retry_state, - grpc_error* error, grpc_core::CallCombinerClosureList* closures) { +void CallData::AddClosuresToFailUnstartedPendingBatches( + grpc_call_element* elem, SubchannelCallRetryState* retry_state, + grpc_error* error, CallCombinerClosureList* closures) { ChannelData* chand = static_cast(elem->channel_data); - call_data* calld = static_cast(elem->call_data); - for (size_t i = 0; i < GPR_ARRAY_SIZE(calld->pending_batches); ++i) { - pending_batch* pending = &calld->pending_batches[i]; - if (pending_batch_is_unstarted(pending, calld, retry_state)) { + for (size_t i = 0; i < GPR_ARRAY_SIZE(pending_batches_); ++i) { + PendingBatch* pending = &pending_batches_[i]; + if (PendingBatchIsUnstarted(pending, retry_state)) { if (grpc_client_channel_call_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: failing unstarted pending batch at index " "%" PRIuPTR, - chand, calld, i); + chand, this, i); } closures->Add(pending->batch->on_complete, GRPC_ERROR_REF(error), "failing on_complete for pending batch"); pending->batch->on_complete = nullptr; - maybe_clear_pending_batch(elem, pending); + MaybeClearPendingBatch(elem, pending); } } GRPC_ERROR_UNREF(error); } -// Runs necessary closures upon completion of a call attempt. -static void run_closures_for_completed_call(subchannel_batch_data* batch_data, - grpc_error* error) { +void CallData::RunClosuresForCompletedCall(SubchannelCallBatchData* batch_data, + grpc_error* error) { grpc_call_element* elem = batch_data->elem; - call_data* calld = static_cast(elem->call_data); - subchannel_call_retry_state* retry_state = - static_cast( + SubchannelCallRetryState* retry_state = + static_cast( batch_data->subchannel_call->GetParentData()); // Construct list of closures to execute. - grpc_core::CallCombinerClosureList closures; + CallCombinerClosureList closures; // First, add closure for recv_trailing_metadata_ready. - add_closure_for_recv_trailing_metadata_ready( - elem, batch_data, GRPC_ERROR_REF(error), &closures); + AddClosureForRecvTrailingMetadataReady(elem, batch_data, + GRPC_ERROR_REF(error), &closures); // If there are deferred recv_initial_metadata_ready or recv_message_ready // callbacks, add them to closures. - add_closures_for_deferred_recv_callbacks(batch_data, retry_state, &closures); + AddClosuresForDeferredRecvCallbacks(batch_data, retry_state, &closures); // Add closures to fail any pending batches that have not yet been started. - add_closures_to_fail_unstarted_pending_batches( - elem, retry_state, GRPC_ERROR_REF(error), &closures); + AddClosuresToFailUnstartedPendingBatches(elem, retry_state, + GRPC_ERROR_REF(error), &closures); // Don't need batch_data anymore. - batch_data_unref(batch_data); + batch_data->Unref(); // Schedule all of the closures identified above. // Note: This will release the call combiner. - closures.RunClosures(calld->call_combiner); + closures.RunClosures(call_combiner_); GRPC_ERROR_UNREF(error); } -// Intercepts recv_trailing_metadata_ready callback for retries. -// Commits the call and returns the trailing metadata up the stack. -static void recv_trailing_metadata_ready(void* arg, grpc_error* error) { - subchannel_batch_data* batch_data = static_cast(arg); +void CallData::RecvTrailingMetadataReady(void* arg, grpc_error* error) { + SubchannelCallBatchData* batch_data = + static_cast(arg); grpc_call_element* elem = batch_data->elem; ChannelData* chand = static_cast(elem->channel_data); - call_data* calld = static_cast(elem->call_data); + CallData* calld = static_cast(elem->call_data); if (grpc_client_channel_call_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: got recv_trailing_metadata_ready, error=%s", chand, calld, grpc_error_string(error)); } - subchannel_call_retry_state* retry_state = - static_cast( + SubchannelCallRetryState* retry_state = + static_cast( batch_data->subchannel_call->GetParentData()); retry_state->completed_recv_trailing_metadata = true; // Get the call's status and check for server pushback metadata. @@ -2179,44 +2391,42 @@ static void recv_trailing_metadata_ready(void* arg, grpc_error* error) { grpc_mdelem* server_pushback_md = nullptr; grpc_metadata_batch* md_batch = batch_data->batch.payload->recv_trailing_metadata.recv_trailing_metadata; - get_call_status(elem, md_batch, GRPC_ERROR_REF(error), &status, - &server_pushback_md); + calld->GetCallStatus(elem, md_batch, GRPC_ERROR_REF(error), &status, + &server_pushback_md); if (grpc_client_channel_call_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: call finished, status=%s", chand, calld, grpc_status_code_to_string(status)); } // Check if we should retry. - if (maybe_retry(elem, batch_data, status, server_pushback_md)) { + if (calld->MaybeRetry(elem, batch_data, status, server_pushback_md)) { // Unref batch_data for deferred recv_initial_metadata_ready or // recv_message_ready callbacks, if any. if (retry_state->recv_initial_metadata_ready_deferred_batch != nullptr) { - batch_data_unref(batch_data); + batch_data->Unref(); GRPC_ERROR_UNREF(retry_state->recv_initial_metadata_error); } if (retry_state->recv_message_ready_deferred_batch != nullptr) { - batch_data_unref(batch_data); + batch_data->Unref(); GRPC_ERROR_UNREF(retry_state->recv_message_error); } - batch_data_unref(batch_data); + batch_data->Unref(); return; } // Not retrying, so commit the call. - retry_commit(elem, retry_state); + calld->RetryCommit(elem, retry_state); // Run any necessary closures. - run_closures_for_completed_call(batch_data, GRPC_ERROR_REF(error)); + calld->RunClosuresForCompletedCall(batch_data, GRPC_ERROR_REF(error)); } // // on_complete callback handling // -// Adds the on_complete closure for the pending batch completed in -// batch_data to closures. -static void add_closure_for_completed_pending_batch( - grpc_call_element* elem, subchannel_batch_data* batch_data, - subchannel_call_retry_state* retry_state, grpc_error* error, - grpc_core::CallCombinerClosureList* closures) { - pending_batch* pending = pending_batch_find( +void CallData::AddClosuresForCompletedPendingBatch( + grpc_call_element* elem, SubchannelCallBatchData* batch_data, + SubchannelCallRetryState* retry_state, grpc_error* error, + CallCombinerClosureList* closures) { + PendingBatch* pending = PendingBatchFind( elem, "completed", [batch_data](grpc_transport_stream_op_batch* batch) { // Match the pending batch with the same set of send ops as the // subchannel batch we've just completed. @@ -2237,27 +2447,22 @@ static void add_closure_for_completed_pending_batch( closures->Add(pending->batch->on_complete, error, "on_complete for pending batch"); pending->batch->on_complete = nullptr; - maybe_clear_pending_batch(elem, pending); + MaybeClearPendingBatch(elem, pending); } -// If there are any cached ops to replay or pending ops to start on the -// subchannel call, adds a closure to closures to invoke -// start_retriable_subchannel_batches(). -static void add_closures_for_replay_or_pending_send_ops( - grpc_call_element* elem, subchannel_batch_data* batch_data, - subchannel_call_retry_state* retry_state, - grpc_core::CallCombinerClosureList* closures) { +void CallData::AddClosuresForReplayOrPendingSendOps( + grpc_call_element* elem, SubchannelCallBatchData* batch_data, + SubchannelCallRetryState* retry_state, CallCombinerClosureList* closures) { ChannelData* chand = static_cast(elem->channel_data); - call_data* calld = static_cast(elem->call_data); bool have_pending_send_message_ops = - retry_state->started_send_message_count < calld->send_messages.size(); + retry_state->started_send_message_count < send_messages_.size(); bool have_pending_send_trailing_metadata_op = - calld->seen_send_trailing_metadata && + seen_send_trailing_metadata_ && !retry_state->started_send_trailing_metadata; if (!have_pending_send_message_ops && !have_pending_send_trailing_metadata_op) { - for (size_t i = 0; i < GPR_ARRAY_SIZE(calld->pending_batches); ++i) { - pending_batch* pending = &calld->pending_batches[i]; + for (size_t i = 0; i < GPR_ARRAY_SIZE(pending_batches_); ++i) { + PendingBatch* pending = &pending_batches_[i]; grpc_transport_stream_op_batch* batch = pending->batch; if (batch == nullptr || pending->send_ops_cached) continue; if (batch->send_message) have_pending_send_message_ops = true; @@ -2270,31 +2475,30 @@ static void add_closures_for_replay_or_pending_send_ops( if (grpc_client_channel_call_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: starting next batch for pending send op(s)", - chand, calld); + chand, this); } GRPC_CLOSURE_INIT(&batch_data->batch.handler_private.closure, - start_retriable_subchannel_batches, elem, + StartRetriableSubchannelBatches, elem, grpc_schedule_on_exec_ctx); closures->Add(&batch_data->batch.handler_private.closure, GRPC_ERROR_NONE, "starting next batch for send_* op(s)"); } } -// Callback used to intercept on_complete from subchannel calls. -// Called only when retries are enabled. -static void on_complete(void* arg, grpc_error* error) { - subchannel_batch_data* batch_data = static_cast(arg); +void CallData::OnComplete(void* arg, grpc_error* error) { + SubchannelCallBatchData* batch_data = + static_cast(arg); grpc_call_element* elem = batch_data->elem; ChannelData* chand = static_cast(elem->channel_data); - call_data* calld = static_cast(elem->call_data); + CallData* calld = static_cast(elem->call_data); if (grpc_client_channel_call_trace.enabled()) { char* batch_str = grpc_transport_stream_op_batch_string(&batch_data->batch); gpr_log(GPR_INFO, "chand=%p calld=%p: got on_complete, error=%s, batch=%s", chand, calld, grpc_error_string(error), batch_str); gpr_free(batch_str); } - subchannel_call_retry_state* retry_state = - static_cast( + SubchannelCallRetryState* retry_state = + static_cast( batch_data->subchannel_call->GetParentData()); // Update bookkeeping in retry_state. if (batch_data->batch.send_initial_metadata) { @@ -2308,38 +2512,38 @@ static void on_complete(void* arg, grpc_error* error) { } // If the call is committed, free cached data for send ops that we've just // completed. - if (calld->retry_committed) { - free_cached_send_op_data_for_completed_batch(elem, batch_data, retry_state); + if (calld->retry_committed_) { + calld->FreeCachedSendOpDataForCompletedBatch(elem, batch_data, retry_state); } // Construct list of closures to execute. - grpc_core::CallCombinerClosureList closures; + CallCombinerClosureList closures; // If a retry was already dispatched, that means we saw // recv_trailing_metadata before this, so we do nothing here. // Otherwise, invoke the callback to return the result to the surface. if (!retry_state->retry_dispatched) { // Add closure for the completed pending batch, if any. - add_closure_for_completed_pending_batch(elem, batch_data, retry_state, - GRPC_ERROR_REF(error), &closures); + calld->AddClosuresForCompletedPendingBatch( + elem, batch_data, retry_state, GRPC_ERROR_REF(error), &closures); // If needed, add a callback to start any replay or pending send ops on // the subchannel call. if (!retry_state->completed_recv_trailing_metadata) { - add_closures_for_replay_or_pending_send_ops(elem, batch_data, retry_state, + calld->AddClosuresForReplayOrPendingSendOps(elem, batch_data, retry_state, &closures); } } // Track number of pending subchannel send batches and determine if this // was the last one. - --calld->num_pending_retriable_subchannel_send_batches; + --calld->num_pending_retriable_subchannel_send_batches_; const bool last_send_batch_complete = - calld->num_pending_retriable_subchannel_send_batches == 0; + calld->num_pending_retriable_subchannel_send_batches_ == 0; // Don't need batch_data anymore. - batch_data_unref(batch_data); + batch_data->Unref(); // Schedule all of the closures identified above. // Note: This yeilds the call combiner. - closures.RunClosures(calld->call_combiner); + closures.RunClosures(calld->call_combiner_); // If this was the last subchannel send batch, unref the call stack. if (last_send_batch_complete) { - GRPC_CALL_STACK_UNREF(calld->owning_call, "subchannel_send_batches"); + GRPC_CALL_STACK_UNREF(calld->owning_call_, "subchannel_send_batches"); } } @@ -2347,40 +2551,35 @@ static void on_complete(void* arg, grpc_error* error) { // subchannel batch construction // -// Helper function used to start a subchannel batch in the call combiner. -static void start_batch_in_call_combiner(void* arg, grpc_error* ignored) { +void CallData::StartBatchInCallCombiner(void* arg, grpc_error* ignored) { grpc_transport_stream_op_batch* batch = static_cast(arg); - grpc_core::SubchannelCall* subchannel_call = - static_cast(batch->handler_private.extra_arg); + SubchannelCall* subchannel_call = + static_cast(batch->handler_private.extra_arg); // Note: This will release the call combiner. subchannel_call->StartTransportStreamOpBatch(batch); } -// Adds a closure to closures that will execute batch in the call combiner. -static void add_closure_for_subchannel_batch( +void CallData::AddClosureForSubchannelBatch( grpc_call_element* elem, grpc_transport_stream_op_batch* batch, - grpc_core::CallCombinerClosureList* closures) { + CallCombinerClosureList* closures) { ChannelData* chand = static_cast(elem->channel_data); - call_data* calld = static_cast(elem->call_data); - batch->handler_private.extra_arg = calld->subchannel_call.get(); - GRPC_CLOSURE_INIT(&batch->handler_private.closure, - start_batch_in_call_combiner, batch, - grpc_schedule_on_exec_ctx); + batch->handler_private.extra_arg = subchannel_call_.get(); + GRPC_CLOSURE_INIT(&batch->handler_private.closure, StartBatchInCallCombiner, + batch, grpc_schedule_on_exec_ctx); if (grpc_client_channel_call_trace.enabled()) { char* batch_str = grpc_transport_stream_op_batch_string(batch); gpr_log(GPR_INFO, "chand=%p calld=%p: starting subchannel batch: %s", chand, - calld, batch_str); + this, batch_str); gpr_free(batch_str); } closures->Add(&batch->handler_private.closure, GRPC_ERROR_NONE, "start_subchannel_batch"); } -// Adds retriable send_initial_metadata op to batch_data. -static void add_retriable_send_initial_metadata_op( - call_data* calld, subchannel_call_retry_state* retry_state, - subchannel_batch_data* batch_data) { +void CallData::AddRetriableSendInitialMetadataOp( + SubchannelCallRetryState* retry_state, + SubchannelCallBatchData* batch_data) { // Maps the number of retries to the corresponding metadata value slice. static const grpc_slice* retry_count_strings[] = { &GRPC_MDSTR_1, &GRPC_MDSTR_2, &GRPC_MDSTR_3, &GRPC_MDSTR_4}; @@ -2390,12 +2589,11 @@ static void add_retriable_send_initial_metadata_op( // // If we've already completed one or more attempts, add the // grpc-retry-attempts header. - retry_state->send_initial_metadata_storage = - static_cast(gpr_arena_alloc( - calld->arena, sizeof(grpc_linked_mdelem) * - (calld->send_initial_metadata.list.count + - (calld->num_attempts_completed > 0)))); - grpc_metadata_batch_copy(&calld->send_initial_metadata, + retry_state->send_initial_metadata_storage = static_cast( + gpr_arena_alloc(arena_, sizeof(grpc_linked_mdelem) * + (send_initial_metadata_.list.count + + (num_attempts_completed_ > 0)))); + grpc_metadata_batch_copy(&send_initial_metadata_, &retry_state->send_initial_metadata, retry_state->send_initial_metadata_storage); if (GPR_UNLIKELY(retry_state->send_initial_metadata.idx.named @@ -2404,14 +2602,14 @@ static void add_retriable_send_initial_metadata_op( retry_state->send_initial_metadata.idx.named .grpc_previous_rpc_attempts); } - if (GPR_UNLIKELY(calld->num_attempts_completed > 0)) { + if (GPR_UNLIKELY(num_attempts_completed_ > 0)) { grpc_mdelem retry_md = grpc_mdelem_create( GRPC_MDSTR_GRPC_PREVIOUS_RPC_ATTEMPTS, - *retry_count_strings[calld->num_attempts_completed - 1], nullptr); + *retry_count_strings[num_attempts_completed_ - 1], nullptr); grpc_error* error = grpc_metadata_batch_add_tail( &retry_state->send_initial_metadata, - &retry_state->send_initial_metadata_storage[calld->send_initial_metadata - .list.count], + &retry_state + ->send_initial_metadata_storage[send_initial_metadata_.list.count], retry_md); if (GPR_UNLIKELY(error != GRPC_ERROR_NONE)) { gpr_log(GPR_ERROR, "error adding retry metadata: %s", @@ -2424,24 +2622,21 @@ static void add_retriable_send_initial_metadata_op( batch_data->batch.payload->send_initial_metadata.send_initial_metadata = &retry_state->send_initial_metadata; batch_data->batch.payload->send_initial_metadata.send_initial_metadata_flags = - calld->send_initial_metadata_flags; - batch_data->batch.payload->send_initial_metadata.peer_string = - calld->peer_string; + send_initial_metadata_flags_; + batch_data->batch.payload->send_initial_metadata.peer_string = peer_string_; } -// Adds retriable send_message op to batch_data. -static void add_retriable_send_message_op( - grpc_call_element* elem, subchannel_call_retry_state* retry_state, - subchannel_batch_data* batch_data) { +void CallData::AddRetriableSendMessageOp(grpc_call_element* elem, + SubchannelCallRetryState* retry_state, + SubchannelCallBatchData* batch_data) { ChannelData* chand = static_cast(elem->channel_data); - call_data* calld = static_cast(elem->call_data); if (grpc_client_channel_call_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: starting calld->send_messages[%" PRIuPTR "]", - chand, calld, retry_state->started_send_message_count); + chand, this, retry_state->started_send_message_count); } - grpc_core::ByteStreamCache* cache = - calld->send_messages[retry_state->started_send_message_count]; + ByteStreamCache* cache = + send_messages_[retry_state->started_send_message_count]; ++retry_state->started_send_message_count; retry_state->send_message.Init(cache); batch_data->batch.send_message = true; @@ -2449,18 +2644,17 @@ static void add_retriable_send_message_op( retry_state->send_message.get()); } -// Adds retriable send_trailing_metadata op to batch_data. -static void add_retriable_send_trailing_metadata_op( - call_data* calld, subchannel_call_retry_state* retry_state, - subchannel_batch_data* batch_data) { +void CallData::AddRetriableSendTrailingMetadataOp( + SubchannelCallRetryState* retry_state, + SubchannelCallBatchData* batch_data) { // We need to make a copy of the metadata batch for each attempt, since // the filters in the subchannel stack may modify this batch, and we don't // want those modifications to be passed forward to subsequent attempts. retry_state->send_trailing_metadata_storage = static_cast(gpr_arena_alloc( - calld->arena, sizeof(grpc_linked_mdelem) * - calld->send_trailing_metadata.list.count)); - grpc_metadata_batch_copy(&calld->send_trailing_metadata, + arena_, + sizeof(grpc_linked_mdelem) * send_trailing_metadata_.list.count)); + grpc_metadata_batch_copy(&send_trailing_metadata_, &retry_state->send_trailing_metadata, retry_state->send_trailing_metadata_storage); retry_state->started_send_trailing_metadata = true; @@ -2469,10 +2663,9 @@ static void add_retriable_send_trailing_metadata_op( &retry_state->send_trailing_metadata; } -// Adds retriable recv_initial_metadata op to batch_data. -static void add_retriable_recv_initial_metadata_op( - call_data* calld, subchannel_call_retry_state* retry_state, - subchannel_batch_data* batch_data) { +void CallData::AddRetriableRecvInitialMetadataOp( + SubchannelCallRetryState* retry_state, + SubchannelCallBatchData* batch_data) { retry_state->started_recv_initial_metadata = true; batch_data->batch.recv_initial_metadata = true; grpc_metadata_batch_init(&retry_state->recv_initial_metadata); @@ -2481,30 +2674,27 @@ static void add_retriable_recv_initial_metadata_op( batch_data->batch.payload->recv_initial_metadata.trailing_metadata_available = &retry_state->trailing_metadata_available; GRPC_CLOSURE_INIT(&retry_state->recv_initial_metadata_ready, - recv_initial_metadata_ready, batch_data, + RecvInitialMetadataReady, batch_data, grpc_schedule_on_exec_ctx); batch_data->batch.payload->recv_initial_metadata.recv_initial_metadata_ready = &retry_state->recv_initial_metadata_ready; } -// Adds retriable recv_message op to batch_data. -static void add_retriable_recv_message_op( - call_data* calld, subchannel_call_retry_state* retry_state, - subchannel_batch_data* batch_data) { +void CallData::AddRetriableRecvMessageOp(SubchannelCallRetryState* retry_state, + SubchannelCallBatchData* batch_data) { ++retry_state->started_recv_message_count; batch_data->batch.recv_message = true; batch_data->batch.payload->recv_message.recv_message = &retry_state->recv_message; - GRPC_CLOSURE_INIT(&retry_state->recv_message_ready, recv_message_ready, + GRPC_CLOSURE_INIT(&retry_state->recv_message_ready, RecvMessageReady, batch_data, grpc_schedule_on_exec_ctx); batch_data->batch.payload->recv_message.recv_message_ready = &retry_state->recv_message_ready; } -// Adds retriable recv_trailing_metadata op to batch_data. -static void add_retriable_recv_trailing_metadata_op( - call_data* calld, subchannel_call_retry_state* retry_state, - subchannel_batch_data* batch_data) { +void CallData::AddRetriableRecvTrailingMetadataOp( + SubchannelCallRetryState* retry_state, + SubchannelCallBatchData* batch_data) { retry_state->started_recv_trailing_metadata = true; batch_data->batch.recv_trailing_metadata = true; grpc_metadata_batch_init(&retry_state->recv_trailing_metadata); @@ -2513,115 +2703,105 @@ static void add_retriable_recv_trailing_metadata_op( batch_data->batch.payload->recv_trailing_metadata.collect_stats = &retry_state->collect_stats; GRPC_CLOSURE_INIT(&retry_state->recv_trailing_metadata_ready, - recv_trailing_metadata_ready, batch_data, + RecvTrailingMetadataReady, batch_data, grpc_schedule_on_exec_ctx); batch_data->batch.payload->recv_trailing_metadata .recv_trailing_metadata_ready = &retry_state->recv_trailing_metadata_ready; - maybe_inject_recv_trailing_metadata_ready_for_lb(calld->pick.pick, - &batch_data->batch); + MaybeInjectRecvTrailingMetadataReadyForLoadBalancingPolicy( + pick_.pick, &batch_data->batch); } -// Helper function used to start a recv_trailing_metadata batch. This -// is used in the case where a recv_initial_metadata or recv_message -// op fails in a way that we know the call is over but when the application -// has not yet started its own recv_trailing_metadata op. -static void start_internal_recv_trailing_metadata(grpc_call_element* elem) { +void CallData::StartInternalRecvTrailingMetadata(grpc_call_element* elem) { ChannelData* chand = static_cast(elem->channel_data); - call_data* calld = static_cast(elem->call_data); if (grpc_client_channel_call_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: call failed but recv_trailing_metadata not " "started; starting it internally", - chand, calld); + chand, this); } - subchannel_call_retry_state* retry_state = - static_cast( - calld->subchannel_call->GetParentData()); + SubchannelCallRetryState* retry_state = + static_cast(subchannel_call_->GetParentData()); // Create batch_data with 2 refs, since this batch will be unreffed twice: // once for the recv_trailing_metadata_ready callback when the subchannel // batch returns, and again when we actually get a recv_trailing_metadata // op from the surface. - subchannel_batch_data* batch_data = - batch_data_create(elem, 2, false /* set_on_complete */); - add_retriable_recv_trailing_metadata_op(calld, retry_state, batch_data); + SubchannelCallBatchData* batch_data = + SubchannelCallBatchData::Create(elem, 2, false /* set_on_complete */); + AddRetriableRecvTrailingMetadataOp(retry_state, batch_data); retry_state->recv_trailing_metadata_internal_batch = batch_data; // Note: This will release the call combiner. - calld->subchannel_call->StartTransportStreamOpBatch(&batch_data->batch); + subchannel_call_->StartTransportStreamOpBatch(&batch_data->batch); } // If there are any cached send ops that need to be replayed on the // current subchannel call, creates and returns a new subchannel batch // to replay those ops. Otherwise, returns nullptr. -static subchannel_batch_data* maybe_create_subchannel_batch_for_replay( - grpc_call_element* elem, subchannel_call_retry_state* retry_state) { +CallData::SubchannelCallBatchData* +CallData::MaybeCreateSubchannelBatchForReplay( + grpc_call_element* elem, SubchannelCallRetryState* retry_state) { ChannelData* chand = static_cast(elem->channel_data); - call_data* calld = static_cast(elem->call_data); - subchannel_batch_data* replay_batch_data = nullptr; + SubchannelCallBatchData* replay_batch_data = nullptr; // send_initial_metadata. - if (calld->seen_send_initial_metadata && + if (seen_send_initial_metadata_ && !retry_state->started_send_initial_metadata && - !calld->pending_send_initial_metadata) { + !pending_send_initial_metadata_) { if (grpc_client_channel_call_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: replaying previously completed " "send_initial_metadata op", - chand, calld); + chand, this); } - replay_batch_data = batch_data_create(elem, 1, true /* set_on_complete */); - add_retriable_send_initial_metadata_op(calld, retry_state, - replay_batch_data); + replay_batch_data = + SubchannelCallBatchData::Create(elem, 1, true /* set_on_complete */); + AddRetriableSendInitialMetadataOp(retry_state, replay_batch_data); } // send_message. // Note that we can only have one send_message op in flight at a time. - if (retry_state->started_send_message_count < calld->send_messages.size() && + if (retry_state->started_send_message_count < send_messages_.size() && retry_state->started_send_message_count == retry_state->completed_send_message_count && - !calld->pending_send_message) { + !pending_send_message_) { if (grpc_client_channel_call_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: replaying previously completed " "send_message op", - chand, calld); + chand, this); } if (replay_batch_data == nullptr) { replay_batch_data = - batch_data_create(elem, 1, true /* set_on_complete */); + SubchannelCallBatchData::Create(elem, 1, true /* set_on_complete */); } - add_retriable_send_message_op(elem, retry_state, replay_batch_data); + AddRetriableSendMessageOp(elem, retry_state, replay_batch_data); } // send_trailing_metadata. // Note that we only add this op if we have no more send_message ops // to start, since we can't send down any more send_message ops after // send_trailing_metadata. - if (calld->seen_send_trailing_metadata && - retry_state->started_send_message_count == calld->send_messages.size() && + if (seen_send_trailing_metadata_ && + retry_state->started_send_message_count == send_messages_.size() && !retry_state->started_send_trailing_metadata && - !calld->pending_send_trailing_metadata) { + !pending_send_trailing_metadata_) { if (grpc_client_channel_call_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: replaying previously completed " "send_trailing_metadata op", - chand, calld); + chand, this); } if (replay_batch_data == nullptr) { replay_batch_data = - batch_data_create(elem, 1, true /* set_on_complete */); + SubchannelCallBatchData::Create(elem, 1, true /* set_on_complete */); } - add_retriable_send_trailing_metadata_op(calld, retry_state, - replay_batch_data); + AddRetriableSendTrailingMetadataOp(retry_state, replay_batch_data); } return replay_batch_data; } -// Adds subchannel batches for pending batches to batches, updating -// *num_batches as needed. -static void add_subchannel_batches_for_pending_batches( - grpc_call_element* elem, subchannel_call_retry_state* retry_state, - grpc_core::CallCombinerClosureList* closures) { - call_data* calld = static_cast(elem->call_data); - for (size_t i = 0; i < GPR_ARRAY_SIZE(calld->pending_batches); ++i) { - pending_batch* pending = &calld->pending_batches[i]; +void CallData::AddSubchannelBatchesForPendingBatches( + grpc_call_element* elem, SubchannelCallRetryState* retry_state, + CallCombinerClosureList* closures) { + for (size_t i = 0; i < GPR_ARRAY_SIZE(pending_batches_); ++i) { + PendingBatch* pending = &pending_batches_[i]; grpc_transport_stream_op_batch* batch = pending->batch; if (batch == nullptr) continue; // Skip any batch that either (a) has already been started on this @@ -2647,7 +2827,7 @@ static void add_subchannel_batches_for_pending_batches( // send_message ops after send_trailing_metadata. if (batch->send_trailing_metadata && (retry_state->started_send_message_count + batch->send_message < - calld->send_messages.size() || + send_messages_.size() || retry_state->started_send_trailing_metadata)) { continue; } @@ -2662,7 +2842,7 @@ static void add_subchannel_batches_for_pending_batches( if (batch->recv_trailing_metadata && retry_state->started_recv_trailing_metadata) { // If we previously completed a recv_trailing_metadata op - // initiated by start_internal_recv_trailing_metadata(), use the + // initiated by StartInternalRecvTrailingMetadata(), use the // result of that instead of trying to re-start this op. if (GPR_UNLIKELY((retry_state->recv_trailing_metadata_internal_batch != nullptr))) { @@ -2678,18 +2858,17 @@ static void add_subchannel_batches_for_pending_batches( "re-executing recv_trailing_metadata_ready to propagate " "internally triggered result"); } else { - batch_data_unref(retry_state->recv_trailing_metadata_internal_batch); + retry_state->recv_trailing_metadata_internal_batch->Unref(); } retry_state->recv_trailing_metadata_internal_batch = nullptr; } continue; } // If we're not retrying, just send the batch as-is. - if (calld->method_params == nullptr || - calld->method_params->retry_policy() == nullptr || - calld->retry_committed) { - add_closure_for_subchannel_batch(elem, batch, closures); - pending_batch_clear(calld, pending); + if (method_params_ == nullptr || + method_params_->retry_policy() == nullptr || retry_committed_) { + AddClosureForSubchannelBatch(elem, batch, closures); + PendingBatchClear(pending); continue; } // Create batch with the right number of callbacks. @@ -2699,183 +2878,168 @@ static void add_subchannel_batches_for_pending_batches( const int num_callbacks = has_send_ops + batch->recv_initial_metadata + batch->recv_message + batch->recv_trailing_metadata; - subchannel_batch_data* batch_data = batch_data_create( + SubchannelCallBatchData* batch_data = SubchannelCallBatchData::Create( elem, num_callbacks, has_send_ops /* set_on_complete */); // Cache send ops if needed. - maybe_cache_send_ops_for_batch(calld, pending); + MaybeCacheSendOpsForBatch(pending); // send_initial_metadata. if (batch->send_initial_metadata) { - add_retriable_send_initial_metadata_op(calld, retry_state, batch_data); + AddRetriableSendInitialMetadataOp(retry_state, batch_data); } // send_message. if (batch->send_message) { - add_retriable_send_message_op(elem, retry_state, batch_data); + AddRetriableSendMessageOp(elem, retry_state, batch_data); } // send_trailing_metadata. if (batch->send_trailing_metadata) { - add_retriable_send_trailing_metadata_op(calld, retry_state, batch_data); + AddRetriableSendTrailingMetadataOp(retry_state, batch_data); } // recv_initial_metadata. if (batch->recv_initial_metadata) { // recv_flags is only used on the server side. GPR_ASSERT(batch->payload->recv_initial_metadata.recv_flags == nullptr); - add_retriable_recv_initial_metadata_op(calld, retry_state, batch_data); + AddRetriableRecvInitialMetadataOp(retry_state, batch_data); } // recv_message. if (batch->recv_message) { - add_retriable_recv_message_op(calld, retry_state, batch_data); + AddRetriableRecvMessageOp(retry_state, batch_data); } // recv_trailing_metadata. if (batch->recv_trailing_metadata) { - add_retriable_recv_trailing_metadata_op(calld, retry_state, batch_data); + AddRetriableRecvTrailingMetadataOp(retry_state, batch_data); } - add_closure_for_subchannel_batch(elem, &batch_data->batch, closures); + AddClosureForSubchannelBatch(elem, &batch_data->batch, closures); // Track number of pending subchannel send batches. // If this is the first one, take a ref to the call stack. if (batch->send_initial_metadata || batch->send_message || batch->send_trailing_metadata) { - if (calld->num_pending_retriable_subchannel_send_batches == 0) { - GRPC_CALL_STACK_REF(calld->owning_call, "subchannel_send_batches"); + if (num_pending_retriable_subchannel_send_batches_ == 0) { + GRPC_CALL_STACK_REF(owning_call_, "subchannel_send_batches"); } - ++calld->num_pending_retriable_subchannel_send_batches; + ++num_pending_retriable_subchannel_send_batches_; } } } -// Constructs and starts whatever subchannel batches are needed on the -// subchannel call. -static void start_retriable_subchannel_batches(void* arg, grpc_error* ignored) { +void CallData::StartRetriableSubchannelBatches(void* arg, grpc_error* ignored) { grpc_call_element* elem = static_cast(arg); ChannelData* chand = static_cast(elem->channel_data); - call_data* calld = static_cast(elem->call_data); + CallData* calld = static_cast(elem->call_data); if (grpc_client_channel_call_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: constructing retriable batches", chand, calld); } - subchannel_call_retry_state* retry_state = - static_cast( - calld->subchannel_call->GetParentData()); + SubchannelCallRetryState* retry_state = + static_cast( + calld->subchannel_call_->GetParentData()); // Construct list of closures to execute, one for each pending batch. - grpc_core::CallCombinerClosureList closures; + CallCombinerClosureList closures; // Replay previously-returned send_* ops if needed. - subchannel_batch_data* replay_batch_data = - maybe_create_subchannel_batch_for_replay(elem, retry_state); + SubchannelCallBatchData* replay_batch_data = + calld->MaybeCreateSubchannelBatchForReplay(elem, retry_state); if (replay_batch_data != nullptr) { - add_closure_for_subchannel_batch(elem, &replay_batch_data->batch, - &closures); + calld->AddClosureForSubchannelBatch(elem, &replay_batch_data->batch, + &closures); // Track number of pending subchannel send batches. // If this is the first one, take a ref to the call stack. - if (calld->num_pending_retriable_subchannel_send_batches == 0) { - GRPC_CALL_STACK_REF(calld->owning_call, "subchannel_send_batches"); + if (calld->num_pending_retriable_subchannel_send_batches_ == 0) { + GRPC_CALL_STACK_REF(calld->owning_call_, "subchannel_send_batches"); } - ++calld->num_pending_retriable_subchannel_send_batches; + ++calld->num_pending_retriable_subchannel_send_batches_; } // Now add pending batches. - add_subchannel_batches_for_pending_batches(elem, retry_state, &closures); + calld->AddSubchannelBatchesForPendingBatches(elem, retry_state, &closures); // Start batches on subchannel call. if (grpc_client_channel_call_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: starting %" PRIuPTR " retriable batches on subchannel_call=%p", - chand, calld, closures.size(), calld->subchannel_call.get()); + chand, calld, closures.size(), calld->subchannel_call_.get()); } // Note: This will yield the call combiner. - closures.RunClosures(calld->call_combiner); + closures.RunClosures(calld->call_combiner_); } // // LB pick // -static void create_subchannel_call(grpc_call_element* elem) { +void CallData::CreateSubchannelCall(grpc_call_element* elem) { ChannelData* chand = static_cast(elem->channel_data); - call_data* calld = static_cast(elem->call_data); const size_t parent_data_size = - calld->enable_retries ? sizeof(subchannel_call_retry_state) : 0; - const grpc_core::ConnectedSubchannel::CallArgs call_args = { - calld->pollent, // pollent - calld->path, // path - calld->call_start_time, // start_time - calld->deadline, // deadline - calld->arena, // arena + enable_retries_ ? sizeof(SubchannelCallRetryState) : 0; + const ConnectedSubchannel::CallArgs call_args = { + pollent_, path_, call_start_time_, deadline_, arena_, // TODO(roth): When we implement hedging support, we will probably // need to use a separate call context for each subchannel call. - calld->call_context, // context - calld->call_combiner, // call_combiner - parent_data_size // parent_data_size - }; + call_context_, call_combiner_, parent_data_size}; grpc_error* error = GRPC_ERROR_NONE; - calld->subchannel_call = - calld->pick.pick.connected_subchannel->CreateCall(call_args, &error); + subchannel_call_ = + pick_.pick.connected_subchannel->CreateCall(call_args, &error); if (grpc_client_channel_routing_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: create subchannel_call=%p: error=%s", - chand, calld, calld->subchannel_call.get(), - grpc_error_string(error)); + chand, this, subchannel_call_.get(), grpc_error_string(error)); } if (GPR_UNLIKELY(error != GRPC_ERROR_NONE)) { - pending_batches_fail(elem, error, yield_call_combiner); + PendingBatchesFail(elem, error, YieldCallCombiner); } else { if (parent_data_size > 0) { - new (calld->subchannel_call->GetParentData()) - subchannel_call_retry_state(calld->call_context); + new (subchannel_call_->GetParentData()) + SubchannelCallRetryState(call_context_); } - pending_batches_resume(elem); + PendingBatchesResume(elem); } } -// Invoked when a pick is completed, on both success or failure. -static void pick_done(void* arg, grpc_error* error) { +void CallData::PickDone(void* arg, grpc_error* error) { grpc_call_element* elem = static_cast(arg); ChannelData* chand = static_cast(elem->channel_data); - call_data* calld = static_cast(elem->call_data); + CallData* calld = static_cast(elem->call_data); if (error != GRPC_ERROR_NONE) { if (grpc_client_channel_routing_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: failed to pick subchannel: error=%s", chand, calld, grpc_error_string(error)); } - pending_batches_fail(elem, GRPC_ERROR_REF(error), yield_call_combiner); + calld->PendingBatchesFail(elem, GRPC_ERROR_REF(error), YieldCallCombiner); return; } - create_subchannel_call(elem); + calld->CreateSubchannelCall(elem); } -namespace grpc_core { -namespace { - // A class to handle the call combiner cancellation callback for a // queued pick. -class QueuedPickCanceller { +class CallData::QueuedPickCanceller { public: explicit QueuedPickCanceller(grpc_call_element* elem) : elem_(elem) { - auto* calld = static_cast(elem->call_data); + auto* calld = static_cast(elem->call_data); auto* chand = static_cast(elem->channel_data); - GRPC_CALL_STACK_REF(calld->owning_call, "QueuedPickCanceller"); + GRPC_CALL_STACK_REF(calld->owning_call_, "QueuedPickCanceller"); GRPC_CLOSURE_INIT(&closure_, &CancelLocked, this, grpc_combiner_scheduler(chand->data_plane_combiner())); - grpc_call_combiner_set_notify_on_cancel(calld->call_combiner, &closure_); + grpc_call_combiner_set_notify_on_cancel(calld->call_combiner_, &closure_); } private: static void CancelLocked(void* arg, grpc_error* error) { auto* self = static_cast(arg); auto* chand = static_cast(self->elem_->channel_data); - auto* calld = static_cast(self->elem_->call_data); + auto* calld = static_cast(self->elem_->call_data); if (grpc_client_channel_routing_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: cancelling queued pick: " "error=%s self=%p calld->pick_canceller=%p", chand, calld, grpc_error_string(error), self, - calld->pick_canceller); + calld->pick_canceller_); } - if (calld->pick_canceller == self && error != GRPC_ERROR_NONE) { + if (calld->pick_canceller_ == self && error != GRPC_ERROR_NONE) { // Remove pick from list of queued picks. - remove_call_from_queued_picks_locked(self->elem_); + calld->RemoveCallFromQueuedPicksLocked(self->elem_); // Fail pending batches on the call. - pending_batches_fail(self->elem_, GRPC_ERROR_REF(error), - yield_call_combiner_if_pending_batches_found); + calld->PendingBatchesFail(self->elem_, GRPC_ERROR_REF(error), + YieldCallCombinerIfPendingBatchesFound); } - GRPC_CALL_STACK_UNREF(calld->owning_call, "QueuedPickCanceller"); + GRPC_CALL_STACK_UNREF(calld->owning_call_, "QueuedPickCanceller"); Delete(self); } @@ -2883,72 +3047,61 @@ class QueuedPickCanceller { grpc_closure closure_; }; -} // namespace -} // namespace grpc_core - -// Removes the call from the channel's list of queued picks. -static void remove_call_from_queued_picks_locked(grpc_call_element* elem) { +void CallData::RemoveCallFromQueuedPicksLocked(grpc_call_element* elem) { auto* chand = static_cast(elem->channel_data); - auto* calld = static_cast(elem->call_data); if (grpc_client_channel_routing_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: removing from queued picks list", - chand, calld); + chand, this); } - chand->RemoveQueuedPick(&calld->pick, calld->pollent); - calld->pick_queued = false; + chand->RemoveQueuedPick(&pick_, pollent_); + pick_queued_ = false; // Lame the call combiner canceller. - calld->pick_canceller = nullptr; + pick_canceller_ = nullptr; } -// Adds the call to the channel's list of queued picks. -static void add_call_to_queued_picks_locked(grpc_call_element* elem) { +void CallData::AddCallToQueuedPicksLocked(grpc_call_element* elem) { auto* chand = static_cast(elem->channel_data); - auto* calld = static_cast(elem->call_data); if (grpc_client_channel_routing_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: adding to queued picks list", chand, - calld); + this); } - calld->pick_queued = true; - calld->pick.elem = elem; - chand->AddQueuedPick(&calld->pick, calld->pollent); + pick_queued_ = true; + pick_.elem = elem; + chand->AddQueuedPick(&pick_, pollent_); // Register call combiner cancellation callback. - calld->pick_canceller = grpc_core::New(elem); + pick_canceller_ = New(elem); } -// Applies service config to the call. Must be invoked once we know -// that the resolver has returned results to the channel. -static void apply_service_config_to_call_locked(grpc_call_element* elem) { +void CallData::ApplyServiceConfigToCallLocked(grpc_call_element* elem) { ChannelData* chand = static_cast(elem->channel_data); - call_data* calld = static_cast(elem->call_data); if (grpc_client_channel_routing_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: applying service config to call", - chand, calld); + chand, this); } - calld->retry_throttle_data = chand->retry_throttle_data(); - calld->method_params = chand->GetMethodParams(calld->path); - if (calld->method_params != nullptr) { + retry_throttle_data_ = chand->retry_throttle_data(); + method_params_ = chand->GetMethodParams(path_); + if (method_params_ != nullptr) { // If the deadline from the service config is shorter than the one // from the client API, reset the deadline timer. - if (chand->deadline_checking_enabled() && - calld->method_params->timeout() != 0) { + if (chand->deadline_checking_enabled() && method_params_->timeout() != 0) { const grpc_millis per_method_deadline = - grpc_timespec_to_millis_round_up(calld->call_start_time) + - calld->method_params->timeout(); - if (per_method_deadline < calld->deadline) { - calld->deadline = per_method_deadline; - grpc_deadline_state_reset(elem, calld->deadline); + grpc_timespec_to_millis_round_up(call_start_time_) + + method_params_->timeout(); + if (per_method_deadline < deadline_) { + deadline_ = per_method_deadline; + grpc_deadline_state_reset(elem, deadline_); } } // If the service config set wait_for_ready and the application // did not explicitly set it, use the value from the service config. uint32_t* send_initial_metadata_flags = - &calld->pending_batches[0] + &pending_batches_[0] .batch->payload->send_initial_metadata.send_initial_metadata_flags; - if (GPR_UNLIKELY(calld->method_params->wait_for_ready() != + if (GPR_UNLIKELY(method_params_->wait_for_ready() != ClientChannelMethodParams::WAIT_FOR_READY_UNSET && !(*send_initial_metadata_flags & GRPC_INITIAL_METADATA_WAIT_FOR_READY_EXPLICITLY_SET))) { - if (calld->method_params->wait_for_ready() == + if (method_params_->wait_for_ready() == ClientChannelMethodParams::WAIT_FOR_READY_TRUE) { *send_initial_metadata_flags |= GRPC_INITIAL_METADATA_WAIT_FOR_READY; } else { @@ -2958,26 +3111,23 @@ static void apply_service_config_to_call_locked(grpc_call_element* elem) { } // If no retry policy, disable retries. // TODO(roth): Remove this when adding support for transparent retries. - if (calld->method_params == nullptr || - calld->method_params->retry_policy() == nullptr) { - calld->enable_retries = false; + if (method_params_ == nullptr || method_params_->retry_policy() == nullptr) { + enable_retries_ = false; } } -// Invoked once resolver results are available. -static void maybe_apply_service_config_to_call_locked(grpc_call_element* elem) { +void CallData::MaybeApplyServiceConfigToCallLocked(grpc_call_element* elem) { ChannelData* chand = static_cast(elem->channel_data); - call_data* calld = static_cast(elem->call_data); // Apply service config data to the call only once, and only if the // channel has the data available. if (GPR_LIKELY(chand->received_service_config_data() && - !calld->service_config_applied)) { - calld->service_config_applied = true; - apply_service_config_to_call_locked(elem); + !service_config_applied_)) { + service_config_applied_ = true; + ApplyServiceConfigToCallLocked(elem); } } -static const char* pick_result_name(LoadBalancingPolicy::PickResult result) { +const char* PickResultName(LoadBalancingPolicy::PickResult result) { switch (result) { case LoadBalancingPolicy::PICK_COMPLETE: return "COMPLETE"; @@ -2989,47 +3139,47 @@ static const char* pick_result_name(LoadBalancingPolicy::PickResult result) { GPR_UNREACHABLE_CODE(return "UNKNOWN"); } -static void start_pick_locked(void* arg, grpc_error* error) { +void CallData::StartPickLocked(void* arg, grpc_error* error) { grpc_call_element* elem = static_cast(arg); - call_data* calld = static_cast(elem->call_data); + CallData* calld = static_cast(elem->call_data); ChannelData* chand = static_cast(elem->channel_data); - GPR_ASSERT(calld->pick.pick.connected_subchannel == nullptr); - GPR_ASSERT(calld->subchannel_call == nullptr); + GPR_ASSERT(calld->pick_.pick.connected_subchannel == nullptr); + GPR_ASSERT(calld->subchannel_call_ == nullptr); // If this is a retry, use the send_initial_metadata payload that // we've cached; otherwise, use the pending batch. The // send_initial_metadata batch will be the first pending batch in the - // list, as set by get_batch_index() above. + // list, as set by GetBatchIndex() above. // TODO(roth): What if the LB policy needs to add something to the // call's initial metadata, and then there's a retry? We don't want // the new metadata to be added twice. We might need to somehow // allocate the subchannel batch earlier so that we can give the // subchannel's copy of the metadata batch (which is copied for each // attempt) to the LB policy instead the one from the parent channel. - calld->pick.pick.initial_metadata = - calld->seen_send_initial_metadata - ? &calld->send_initial_metadata - : calld->pending_batches[0] + calld->pick_.pick.initial_metadata = + calld->seen_send_initial_metadata_ + ? &calld->send_initial_metadata_ + : calld->pending_batches_[0] .batch->payload->send_initial_metadata.send_initial_metadata; uint32_t* send_initial_metadata_flags = - calld->seen_send_initial_metadata - ? &calld->send_initial_metadata_flags - : &calld->pending_batches[0] + calld->seen_send_initial_metadata_ + ? &calld->send_initial_metadata_flags_ + : &calld->pending_batches_[0] .batch->payload->send_initial_metadata .send_initial_metadata_flags; // Apply service config to call if needed. - maybe_apply_service_config_to_call_locked(elem); + calld->MaybeApplyServiceConfigToCallLocked(elem); // When done, we schedule this closure to leave the data plane combiner. - GRPC_CLOSURE_INIT(&calld->pick_closure, pick_done, elem, + GRPC_CLOSURE_INIT(&calld->pick_closure_, PickDone, elem, grpc_schedule_on_exec_ctx); // Attempt pick. error = GRPC_ERROR_NONE; - auto pick_result = chand->picker()->Pick(&calld->pick.pick, &error); + auto pick_result = chand->picker()->Pick(&calld->pick_.pick, &error); if (grpc_client_channel_routing_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: LB pick returned %s (connected_subchannel=%p, " "error=%s)", - chand, calld, pick_result_name(pick_result), - calld->pick.pick.connected_subchannel.get(), + chand, calld, PickResultName(pick_result), + calld->pick_.pick.connected_subchannel.get(), grpc_error_string(error)); } switch (pick_result) { @@ -3038,7 +3188,7 @@ static void start_pick_locked(void* arg, grpc_error* error) { grpc_error* disconnect_error = chand->disconnect_error(); if (disconnect_error != GRPC_ERROR_NONE) { GRPC_ERROR_UNREF(error); - GRPC_CLOSURE_SCHED(&calld->pick_closure, + GRPC_CLOSURE_SCHED(&calld->pick_closure_, GRPC_ERROR_REF(disconnect_error)); break; } @@ -3048,18 +3198,18 @@ static void start_pick_locked(void* arg, grpc_error* error) { GRPC_INITIAL_METADATA_WAIT_FOR_READY) == 0) { // Retry if appropriate; otherwise, fail. grpc_status_code status = GRPC_STATUS_OK; - grpc_error_get_status(error, calld->deadline, &status, nullptr, nullptr, - nullptr); - if (!calld->enable_retries || - !maybe_retry(elem, nullptr /* batch_data */, status, - nullptr /* server_pushback_md */)) { + grpc_error_get_status(error, calld->deadline_, &status, nullptr, + nullptr, nullptr); + if (!calld->enable_retries_ || + !calld->MaybeRetry(elem, nullptr /* batch_data */, status, + nullptr /* server_pushback_md */)) { grpc_error* new_error = GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING( "Failed to pick subchannel", &error, 1); GRPC_ERROR_UNREF(error); - GRPC_CLOSURE_SCHED(&calld->pick_closure, new_error); + GRPC_CLOSURE_SCHED(&calld->pick_closure_, new_error); } - if (calld->pick_queued) remove_call_from_queued_picks_locked(elem); + if (calld->pick_queued_) calld->RemoveCallFromQueuedPicksLocked(elem); break; } // If wait_for_ready is true, then queue to retry when we get a new @@ -3068,151 +3218,36 @@ static void start_pick_locked(void* arg, grpc_error* error) { } // Fallthrough case LoadBalancingPolicy::PICK_QUEUE: - if (!calld->pick_queued) add_call_to_queued_picks_locked(elem); + if (!calld->pick_queued_) calld->AddCallToQueuedPicksLocked(elem); break; default: // PICK_COMPLETE // Handle drops. - if (GPR_UNLIKELY(calld->pick.pick.connected_subchannel == nullptr)) { + if (GPR_UNLIKELY(calld->pick_.pick.connected_subchannel == nullptr)) { error = GRPC_ERROR_CREATE_FROM_STATIC_STRING( "Call dropped by load balancing policy"); } - GRPC_CLOSURE_SCHED(&calld->pick_closure, error); - if (calld->pick_queued) remove_call_from_queued_picks_locked(elem); + GRPC_CLOSURE_SCHED(&calld->pick_closure_, error); + if (calld->pick_queued_) calld->RemoveCallFromQueuedPicksLocked(elem); } } -// -// filter call vtable functions -// - -static void cc_start_transport_stream_op_batch( - grpc_call_element* elem, grpc_transport_stream_op_batch* batch) { - GPR_TIMER_SCOPE("cc_start_transport_stream_op_batch", 0); - call_data* calld = static_cast(elem->call_data); - ChannelData* chand = static_cast(elem->channel_data); - if (GPR_LIKELY(chand->deadline_checking_enabled())) { - grpc_deadline_state_client_start_transport_stream_op_batch(elem, batch); - } - // If we've previously been cancelled, immediately fail any new batches. - if (GPR_UNLIKELY(calld->cancel_error != GRPC_ERROR_NONE)) { - if (grpc_client_channel_call_trace.enabled()) { - gpr_log(GPR_INFO, "chand=%p calld=%p: failing batch with error: %s", - chand, calld, grpc_error_string(calld->cancel_error)); - } - // Note: This will release the call combiner. - grpc_transport_stream_op_batch_finish_with_failure( - batch, GRPC_ERROR_REF(calld->cancel_error), calld->call_combiner); - return; - } - // Handle cancellation. - if (GPR_UNLIKELY(batch->cancel_stream)) { - // Stash a copy of cancel_error in our call data, so that we can use - // it for subsequent operations. This ensures that if the call is - // cancelled before any batches are passed down (e.g., if the deadline - // is in the past when the call starts), we can return the right - // error to the caller when the first batch does get passed down. - GRPC_ERROR_UNREF(calld->cancel_error); - calld->cancel_error = - GRPC_ERROR_REF(batch->payload->cancel_stream.cancel_error); - if (grpc_client_channel_call_trace.enabled()) { - gpr_log(GPR_INFO, "chand=%p calld=%p: recording cancel_error=%s", chand, - calld, grpc_error_string(calld->cancel_error)); - } - // If we do not have a subchannel call (i.e., a pick has not yet - // been started), fail all pending batches. Otherwise, send the - // cancellation down to the subchannel call. - if (calld->subchannel_call == nullptr) { - // TODO(roth): If there is a pending retry callback, do we need to - // cancel it here? - pending_batches_fail(elem, GRPC_ERROR_REF(calld->cancel_error), - no_yield_call_combiner); - // Note: This will release the call combiner. - grpc_transport_stream_op_batch_finish_with_failure( - batch, GRPC_ERROR_REF(calld->cancel_error), calld->call_combiner); - } else { - // Note: This will release the call combiner. - calld->subchannel_call->StartTransportStreamOpBatch(batch); - } - return; - } - // Add the batch to the pending list. - pending_batches_add(elem, batch); - // Check if we've already gotten a subchannel call. - // Note that once we have completed the pick, we do not need to enter - // the channel combiner, which is more efficient (especially for - // streaming calls). - if (calld->subchannel_call != nullptr) { - if (grpc_client_channel_call_trace.enabled()) { - gpr_log(GPR_INFO, - "chand=%p calld=%p: starting batch on subchannel_call=%p", chand, - calld, calld->subchannel_call.get()); - } - pending_batches_resume(elem); - return; - } - // We do not yet have a subchannel call. - // For batches containing a send_initial_metadata op, enter the channel - // combiner to start a pick. - if (GPR_LIKELY(batch->send_initial_metadata)) { - if (grpc_client_channel_call_trace.enabled()) { - gpr_log(GPR_INFO, "chand=%p calld=%p: entering client_channel combiner", - chand, calld); - } - GRPC_CLOSURE_SCHED( - GRPC_CLOSURE_INIT( - &batch->handler_private.closure, start_pick_locked, elem, - grpc_combiner_scheduler(chand->data_plane_combiner())), - GRPC_ERROR_NONE); - } else { - // For all other batches, release the call combiner. - if (grpc_client_channel_call_trace.enabled()) { - gpr_log(GPR_INFO, - "chand=%p calld=%p: saved batch, yielding call combiner", chand, - calld); - } - GRPC_CALL_COMBINER_STOP(calld->call_combiner, - "batch does not include send_initial_metadata"); - } -} - -/* Constructor for call_data */ -static grpc_error* cc_init_call_elem(grpc_call_element* elem, - const grpc_call_element_args* args) { - ChannelData* chand = static_cast(elem->channel_data); - new (elem->call_data) call_data(elem, *chand, *args); - return GRPC_ERROR_NONE; -} - -/* Destructor for call_data */ -static void cc_destroy_call_elem(grpc_call_element* elem, - const grpc_call_final_info* final_info, - grpc_closure* then_schedule_closure) { - call_data* calld = static_cast(elem->call_data); - if (GPR_LIKELY(calld->subchannel_call != nullptr)) { - calld->subchannel_call->SetAfterCallStackDestroy(then_schedule_closure); - then_schedule_closure = nullptr; - } - calld->~call_data(); - GRPC_CLOSURE_SCHED(then_schedule_closure, GRPC_ERROR_NONE); -} - -static void cc_set_pollset_or_pollset_set(grpc_call_element* elem, - grpc_polling_entity* pollent) { - call_data* calld = static_cast(elem->call_data); - calld->pollent = pollent; -} +} // namespace +} // namespace grpc_core /************************************************************************* * EXPORTED SYMBOLS */ +using grpc_core::CallData; +using grpc_core::ChannelData; + const grpc_channel_filter grpc_client_channel_filter = { - cc_start_transport_stream_op_batch, + CallData::StartTransportStreamOpBatch, ChannelData::StartTransportOp, - sizeof(call_data), - cc_init_call_elem, - cc_set_pollset_or_pollset_set, - cc_destroy_call_elem, + sizeof(CallData), + CallData::Init, + CallData::SetPollent, + CallData::Destroy, sizeof(ChannelData), ChannelData::Init, ChannelData::Destroy, @@ -3257,6 +3292,6 @@ void grpc_client_channel_watch_connectivity_state( grpc_core::RefCountedPtr grpc_client_channel_get_subchannel_call(grpc_call_element* elem) { - call_data* calld = static_cast(elem->call_data); - return calld->subchannel_call; + auto* calld = static_cast(elem->call_data); + return calld->subchannel_call(); } From d48b0d2879b62b02c747fdac814c65fcb3c30de2 Mon Sep 17 00:00:00 2001 From: Moiz Haidry Date: Fri, 19 Apr 2019 10:01:50 -0700 Subject: [PATCH 13/13] Created consolidated xds picker that stored and picks from the locality pickers --- .../client_channel/lb_policy/xds/xds.cc | 161 ++++++++++++++++-- 1 file changed, 143 insertions(+), 18 deletions(-) diff --git a/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc b/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc index dd782ac53c3..000de59448b 100644 --- a/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc +++ b/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc @@ -118,6 +118,7 @@ namespace { constexpr char kXds[] = "xds_experimental"; constexpr char kDefaultLocalityName[] = "xds_default_locality"; +constexpr uint32_t kDefaultLocalityWeight = 3; class XdsLb : public LoadBalancingPolicy { public: @@ -259,26 +260,51 @@ class XdsLb : public LoadBalancingPolicy { bool retry_timer_callback_pending_ = false; }; + // Since pickers are UniquePtrs we use this RefCounted wrapper + // to control references to it by the xds picker and the locality + // entry + class PickerRef : public RefCounted { + public: + explicit PickerRef(UniquePtr picker) + : picker_(std::move(picker)) {} + PickResult Pick(PickArgs* pick, grpc_error** error) { + return picker_->Pick(pick, error); + } + + private: + UniquePtr picker_; + }; + + // The picker will use a stateless weighting algorithm to pick the locality to + // use for each request. class Picker : public SubchannelPicker { public: - Picker(UniquePtr child_picker, - RefCountedPtr client_stats) - : child_picker_(std::move(child_picker)), - client_stats_(std::move(client_stats)) {} + // Maintains a weighted list of pickers from each locality that is in ready + // state. The first element in the pair represents the end of a range + // proportional to the locality's weight. The start of the range is the + // previous value in the vector and is 0 for the first element. + using PickerList = + InlinedVector>, 1>; + Picker(RefCountedPtr client_stats, PickerList pickers) + : client_stats_(std::move(client_stats)), + pickers_(std::move(pickers)) {} PickResult Pick(PickArgs* pick, grpc_error** error) override; private: - UniquePtr child_picker_; + // Calls the picker of the locality that the key falls within + PickResult PickFromLocality(const uint32_t key, PickArgs* pick, + grpc_error** error); RefCountedPtr client_stats_; + PickerList pickers_; }; class LocalityMap { public: class LocalityEntry : public InternallyRefCounted { public: - explicit LocalityEntry(RefCountedPtr parent) - : parent_(std::move(parent)) {} + LocalityEntry(RefCountedPtr parent, uint32_t locality_weight) + : parent_(std::move(parent)), locality_weight_(locality_weight) {} ~LocalityEntry() = default; void UpdateLocked(xds_grpclb_serverlist* serverlist, @@ -323,6 +349,9 @@ class XdsLb : public LoadBalancingPolicy { // pending_child_policy_. Mutex child_policy_mu_; RefCountedPtr parent_; + RefCountedPtr picker_ref_; + grpc_connectivity_state connectivity_state_; + uint32_t locality_weight_; }; void UpdateLocked(const LocalityList& locality_list, @@ -346,7 +375,9 @@ class XdsLb : public LoadBalancingPolicy { gpr_free(locality_name); xds_grpclb_destroy_serverlist(serverlist); } + char* locality_name; + uint32_t locality_weight; // The deserialized response from the balancer. May be nullptr until one // such response has arrived. xds_grpclb_serverlist* serverlist; @@ -412,6 +443,8 @@ class XdsLb : public LoadBalancingPolicy { RefCountedPtr child_policy_config_; // Map of policies to use in the backend LocalityMap locality_map_; + // TODO(mhaidry) : Add support for multiple maps of localities + // with different priorities LocalityList locality_serverlist_; // TODO(mhaidry) : Add a pending locality map that may be swapped with the // the current one when new localities in the pending map are ready @@ -424,8 +457,12 @@ class XdsLb : public LoadBalancingPolicy { XdsLb::PickResult XdsLb::Picker::Pick(PickArgs* pick, grpc_error** error) { // TODO(roth): Add support for drop handling. - // Forward pick to child policy. - PickResult result = child_picker_->Pick(pick, error); + // Generate a random number between 0 and the total weight + const uint32_t key = + (rand() * pickers_[pickers_.size() - 1].first) / RAND_MAX; + // Forward pick to whichever locality maps to the range in which the + // random number falls in. + PickResult result = PickFromLocality(key, pick, error); // If pick succeeded, add client stats. if (result == PickResult::PICK_COMPLETE && pick->connected_subchannel != nullptr && client_stats_ != nullptr) { @@ -434,6 +471,29 @@ XdsLb::PickResult XdsLb::Picker::Pick(PickArgs* pick, grpc_error** error) { return result; } +XdsLb::PickResult XdsLb::Picker::PickFromLocality(const uint32_t key, + PickArgs* pick, + grpc_error** error) { + size_t mid = 0; + size_t start_index = 0; + size_t end_index = pickers_.size() - 1; + size_t index = 0; + while (end_index > start_index) { + mid = (start_index + end_index) / 2; + if (pickers_[mid].first > key) { + end_index = mid; + } else if (pickers_[mid].first < key) { + start_index = mid + 1; + } else { + index = mid + 1; + break; + } + } + if (index == 0) index = start_index; + GPR_ASSERT(pickers_[index].first > key); + return pickers_[index].second->Pick(pick, error); +} + // // serverlist parsing code // @@ -935,6 +995,8 @@ void XdsLb::BalancerChannelState::BalancerCallState:: MakeUnique()); xdslb_policy->locality_serverlist_[0]->locality_name = static_cast(gpr_strdup(kDefaultLocalityName)); + xdslb_policy->locality_serverlist_[0]->locality_weight = + kDefaultLocalityWeight; } // and update the copy in the XdsLb instance. This // serverlist instance will be destroyed either upon the next @@ -1316,8 +1378,8 @@ void XdsLb::LocalityMap::UpdateLocked( gpr_strdup(locality_serverlist[i]->locality_name)); auto iter = map_.find(locality_name); if (iter == map_.end()) { - OrphanablePtr new_entry = - MakeOrphanable(parent->Ref()); + OrphanablePtr new_entry = MakeOrphanable( + parent->Ref(), locality_serverlist[i]->locality_weight); MutexLock lock(&child_refs_mu_); iter = map_.emplace(std::move(locality_name), std::move(new_entry)).first; } @@ -1335,8 +1397,8 @@ void grpc_core::XdsLb::LocalityMap::ShutdownLocked() { } void grpc_core::XdsLb::LocalityMap::ResetBackoffLocked() { - for (auto iter = map_.begin(); iter != map_.end(); iter++) { - iter->second->ResetBackoffLocked(); + for (auto& p : map_) { + p.second->ResetBackoffLocked(); } } @@ -1344,8 +1406,8 @@ void grpc_core::XdsLb::LocalityMap::FillChildRefsForChannelz( channelz::ChildRefsList* child_subchannels, channelz::ChildRefsList* child_channels) { MutexLock lock(&child_refs_mu_); - for (auto iter = map_.begin(); iter != map_.end(); iter++) { - iter->second->FillChildRefsForChannelz(child_subchannels, child_channels); + for (auto& p : map_) { + p.second->FillChildRefsForChannelz(child_subchannels, child_channels); } } @@ -1617,9 +1679,72 @@ void XdsLb::LocalityMap::LocalityEntry::Helper::UpdateState( entry_->parent_->lb_chand_->lb_calld() == nullptr ? nullptr : entry_->parent_->lb_chand_->lb_calld()->client_stats(); - entry_->parent_->channel_control_helper()->UpdateState( - state, UniquePtr( - New(std::move(picker), std::move(client_stats)))); + // Cache the picker and its state in the entry + entry_->picker_ref_ = MakeRefCounted(std::move(picker)); + entry_->connectivity_state_ = state; + // Construct a new xds picker which maintains a map of all locality pickers + // that are ready. Each locality is represented by a portion of the range + // proportional to its weight, such that the total range is the sum of the + // weights of all localities + uint32_t end = 0; + size_t num_connecting = 0; + size_t num_idle = 0; + size_t num_transient_failures = 0; + auto& locality_map = this->entry_->parent_->locality_map_.map_; + Picker::PickerList pickers; + for (auto& p : locality_map) { + const LocalityEntry* entry = p.second.get(); + grpc_connectivity_state connectivity_state = entry->connectivity_state_; + switch (connectivity_state) { + case GRPC_CHANNEL_READY: { + end += entry->locality_weight_; + pickers.push_back(MakePair(end, entry->picker_ref_)); + break; + } + case GRPC_CHANNEL_CONNECTING: { + num_connecting++; + break; + } + case GRPC_CHANNEL_IDLE: { + num_idle++; + break; + } + case GRPC_CHANNEL_TRANSIENT_FAILURE: { + num_transient_failures++; + break; + } + default: { + gpr_log(GPR_ERROR, "Invalid locality connectivity state - %d", + connectivity_state); + } + } + } + // Pass on the constructed xds picker if it has any ready pickers in their map + // otherwise pass a QueuePicker if any of the locality pickers are in a + // connecting or idle state, finally return a transient failure picker if all + // locality pickers are in transient failure + if (pickers.size() > 0) { + entry_->parent_->channel_control_helper()->UpdateState( + GRPC_CHANNEL_READY, + UniquePtr( + New(std::move(client_stats), std::move(pickers)))); + } else if (num_connecting > 0) { + entry_->parent_->channel_control_helper()->UpdateState( + GRPC_CHANNEL_CONNECTING, + UniquePtr(New(this->entry_->parent_))); + } else if (num_idle > 0) { + entry_->parent_->channel_control_helper()->UpdateState( + GRPC_CHANNEL_IDLE, + UniquePtr(New(this->entry_->parent_))); + } else { + GPR_ASSERT(num_transient_failures == locality_map.size()); + grpc_error* error = + grpc_error_set_int(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "connections to all localities failing"), + GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE); + entry_->parent_->channel_control_helper()->UpdateState( + state, UniquePtr(New(error))); + } } void XdsLb::LocalityMap::LocalityEntry::Helper::RequestReresolution() {