From 61360c754b70baa9db1fc8bfcdaff0bb75871abe Mon Sep 17 00:00:00 2001 From: Yushiro FURUKAWA Date: Mon, 10 Feb 2020 23:53:33 +0900 Subject: [PATCH 01/80] Remove trailing-spaces from doc/* --- doc/core/combiner-explainer.md | 10 +++---- doc/core/epoll-polling-engine.md | 2 +- ...grpc-client-server-polling-engine-usage.md | 2 +- doc/core/grpc-cq.md | 2 +- doc/core/moving-to-c++.md | 6 ++--- doc/environment_variables.md | 2 +- doc/fork_support.md | 2 +- doc/http2-interop-test-descriptions.md | 26 +++++++++---------- doc/internationalization.md | 2 +- doc/interop-test-descriptions.md | 10 +++---- doc/keepalive.md | 2 +- doc/python/server_reflection.md | 4 +-- doc/security_audit.md | 8 +++--- doc/unit_testing.md | 2 +- doc/versioning.md | 4 +-- 15 files changed, 42 insertions(+), 42 deletions(-) diff --git a/doc/core/combiner-explainer.md b/doc/core/combiner-explainer.md index 9e9d077273e..20b7f8df189 100644 --- a/doc/core/combiner-explainer.md +++ b/doc/core/combiner-explainer.md @@ -41,9 +41,9 @@ Instead, get a new property: class combiner { mpscq q; // multi-producer single-consumer queue can be made non-blocking state s; // is it empty or executing - + run(f) { - if (q.push(f)) { + if (q.push(f)) { // q.push returns true if it's the first thing while (q.pop(&f)) { // modulo some extra work to avoid races f(); @@ -73,9 +73,9 @@ class combiner { mpscq q; // multi-producer single-consumer queue can be made non-blocking state s; // is it empty or executing queue finally; // you can only do run_finally when you are already running something from the combiner - + run(f) { - if (q.push(f)) { + if (q.push(f)) { // q.push returns true if it's the first thing loop: while (q.pop(&f)) { // modulo some extra work to avoid races @@ -127,7 +127,7 @@ tries to spray events onto as many threads as possible to get as much concurrenc So `offload` really does: -``` +``` workqueue.run(continue_from_while_loop); break; ``` diff --git a/doc/core/epoll-polling-engine.md b/doc/core/epoll-polling-engine.md index e660a709384..8ce54a020d4 100644 --- a/doc/core/epoll-polling-engine.md +++ b/doc/core/epoll-polling-engine.md @@ -104,7 +104,7 @@ void grpc_use_signal(int signal_num) If the calling application does not provide a signal number, then the gRPC library will relegate to using a model similar to the current implementation (where every thread does a blocking `poll()` on its `wakeup_fd` and the `epoll_fd`). The function` psi_wait() `in figure 2 implements this logic. -**>> **(**NOTE**: Or alternatively, we can implement a turnstile polling (i.e having only one thread calling `epoll_wait()` on the epoll set at any time - which all other threads call poll on their `wakeup_fds`) +**>> **(**NOTE**: Or alternatively, we can implement a turnstile polling (i.e having only one thread calling `epoll_wait()` on the epoll set at any time - which all other threads call poll on their `wakeup_fds`) in case of not getting a signal number from the applications. diff --git a/doc/core/grpc-client-server-polling-engine-usage.md b/doc/core/grpc-client-server-polling-engine-usage.md index f66dcf09caa..3aa3cc16a78 100644 --- a/doc/core/grpc-client-server-polling-engine-usage.md +++ b/doc/core/grpc-client-server-polling-engine-usage.md @@ -7,7 +7,7 @@ This document talks about how polling engine is used in gRPC core (both on clien ## gRPC client -### Relation between Call, Channel (sub-channels), Completion queue, `grpc_pollset` +### Relation between Call, Channel (sub-channels), Completion queue, `grpc_pollset` - A gRPC Call is tied to a channel (more specifically a sub-channel) and a completion queue for the lifetime of the call. - Once a _sub-channel_ is picked for the call, the file-descriptor (socket fd in case of TCP channels) is added to the pollset corresponding to call's completion queue. (Recall that as per [grpc-cq](grpc-cq.md), a completion queue has a pollset by default) diff --git a/doc/core/grpc-cq.md b/doc/core/grpc-cq.md index b485c354566..e8338d880bb 100644 --- a/doc/core/grpc-cq.md +++ b/doc/core/grpc-cq.md @@ -61,4 +61,4 @@ grpc_cq_end_op(cq, tag) { } ``` - + diff --git a/doc/core/moving-to-c++.md b/doc/core/moving-to-c++.md index 49daf61aae3..410435c46a6 100644 --- a/doc/core/moving-to-c++.md +++ b/doc/core/moving-to-c++.md @@ -11,7 +11,7 @@ gRPC core was originally written in C89 for several reasons support, etc). Over time, this was changed to C99 as all relevant compilers in active use came to support C99 effectively. -gRPC started allowing to use C++ with a couple of exceptions not to +gRPC started allowing to use C++ with a couple of exceptions not to have C++ library linked such as `libstdc++.so`. (For more detail, see the [proposal](https://github.com/grpc/proposal/blob/master/L6-core-allow-cpp.md)) @@ -25,12 +25,12 @@ C++ compatible with ## Constraints -- Most of features available in C++11 are allowed to use but there are some exceptions +- Most of features available in C++11 are allowed to use but there are some exceptions because gRPC should support old systems. - Should be built with gcc 4.8, clang 3.3, and Visual C++ 2015. - Should be run on Linux system with libstdc++ 6.0.9 to support [manylinux1](https://www.python.org/dev/peps/pep-0513). -- This would limit us not to use modern C++11 standard library such as `filesystem`. +- This would limit us not to use modern C++11 standard library such as `filesystem`. You can easily see whether PR is free from this issue by checking the result of `Artifact Build Linux` test. - `thread_local` is not allowed to use on Apple's products because their old OSes diff --git a/doc/environment_variables.md b/doc/environment_variables.md index aef14a9842c..6b474da9599 100644 --- a/doc/environment_variables.md +++ b/doc/environment_variables.md @@ -69,7 +69,7 @@ some configuration as environment variables that can be set. completion queue - pick_first - traces the pick first load balancing policy - plugin_credentials - traces plugin credentials - - pollable_refcount - traces reference counting of 'pollable' objects (only + - pollable_refcount - traces reference counting of 'pollable' objects (only in DEBUG) - resource_quota - trace resource quota objects internals - round_robin - traces the round_robin load balancing policy diff --git a/doc/fork_support.md b/doc/fork_support.md index d0f59f25da3..ff1018ab2ea 100644 --- a/doc/fork_support.md +++ b/doc/fork_support.md @@ -25,7 +25,7 @@ A regression was noted in cases where users are doing fork/exec. This was due to ```pthread_atfork()``` handler that was added in 1.7 to partially support forking in gRPC. A deadlock can happen when pthread_atfork handler is running, and an application thread is calling into gRPC. -We have provided a workaround for this issue by allowing users to turn +We have provided a workaround for this issue by allowing users to turn off the handler using env flag ```GRPC_ENABLE_FORK_SUPPORT=False```. This should be set whenever a user expects to always call exec immediately following fork. It will disable the fork handlers. diff --git a/doc/http2-interop-test-descriptions.md b/doc/http2-interop-test-descriptions.md index 435a8de709c..c596f7c1637 100644 --- a/doc/http2-interop-test-descriptions.md +++ b/doc/http2-interop-test-descriptions.md @@ -8,7 +8,7 @@ Server ------ The code for the custom http2 server can be found [here](https://github.com/grpc/grpc/tree/master/test/http2_test). -It is responsible for handling requests and sending responses, and also for +It is responsible for handling requests and sending responses, and also for fulfilling the behavior of each particular test case. Server should accept these arguments: @@ -51,7 +51,7 @@ the user application having to do a thing. Client Procedure: 1. Client sends two UnaryCall requests (and sleeps for 1 second in-between). TODO: resolve [9300](https://github.com/grpc/grpc/issues/9300) and remove the 1 second sleep - + ``` { response_size: 314159 @@ -78,7 +78,7 @@ RST_STREAM immediately after sending headers to the client. Procedure: 1. Client sends UnaryCall with: - + ``` { response_size: 314159 @@ -93,7 +93,7 @@ Client asserts: Server Procedure: 1. Server sends a RST_STREAM with error code 0 after sending headers to the client. - + *At the moment the error code and message returned are not standardized throughout all languages. Those checks will be added once all client languages behave the same way. [#9142](https://github.com/grpc/grpc/issues/9142) is in flight.* @@ -104,7 +104,7 @@ RST_STREAM halfway through sending data to the client. Procedure: 1. Client sends UnaryCall with: - + ``` { response_size: 314159 @@ -118,7 +118,7 @@ Client asserts: * Call was not successful. Server Procedure: - 1. Server sends a RST_STREAM with error code 0 after sending half of + 1. Server sends a RST_STREAM with error code 0 after sending half of the requested data to the client. ### rst_after_data @@ -128,7 +128,7 @@ RST_STREAM after sending all of the data to the client. Procedure: 1. Client sends UnaryCall with: - + ``` { response_size: 314159 @@ -156,7 +156,7 @@ server. Procedure: 1. Client sends UnaryCall with: - + ``` { response_size: 314159 @@ -165,16 +165,16 @@ Procedure: } } ``` - + Client asserts: * call was successful. * response payload body is 314159 bytes in size. Server Procedure: - 1. Server tracks the number of outstanding pings (i.e. +1 when it sends a ping, and -1 + 1. Server tracks the number of outstanding pings (i.e. +1 when it sends a ping, and -1 when it receives an ack from the client). 2. Server sends pings before and after sending headers, also before and after sending data. - + Server Asserts: * Number of outstanding pings is 0 when the connection is lost. @@ -185,10 +185,10 @@ This test verifies that the client observes the MAX_CONCURRENT_STREAMS limit set Client Procedure: 1. Client sends initial UnaryCall to allow the server to update its MAX_CONCURRENT_STREAMS settings. 2. Client concurrently sends 10 UnaryCalls. - + Client Asserts: * All UnaryCalls were successful, and had the correct type and payload size. - + Server Procedure: 1. Sets MAX_CONCURRENT_STREAMS to one after the connection is made. diff --git a/doc/internationalization.md b/doc/internationalization.md index 1b614cbd264..6ff5894c8ba 100644 --- a/doc/internationalization.md +++ b/doc/internationalization.md @@ -1,7 +1,7 @@ gRPC Internationalization ========================= -As a universal RPC framework, gRPC needs to be fully usable within/across different international environments. +As a universal RPC framework, gRPC needs to be fully usable within/across different international environments. This document describes gRPC API and behavior specifics when used in a non-english environment. ## API Concepts diff --git a/doc/interop-test-descriptions.md b/doc/interop-test-descriptions.md index 978c422d470..f7fbb48148a 100755 --- a/doc/interop-test-descriptions.md +++ b/doc/interop-test-descriptions.md @@ -1007,21 +1007,21 @@ languages. Therefore they are not part of our interop matrix. #### rpc_soak -The client performs many large_unary RPCs in sequence over the same channel. +The client performs many large_unary RPCs in sequence over the same channel. The number of RPCs is configured by the experimental flag, `soak_iterations`. #### channel_soak -The client performs many large_unary RPCs in sequence. Before each RPC, it -tears down and rebuilds the channel. The number of RPCs is configured by +The client performs many large_unary RPCs in sequence. Before each RPC, it +tears down and rebuilds the channel. The number of RPCs is configured by the experimental flag, `soak_iterations`. -This tests puts stress on several gRPC components; the resolver, the load +This tests puts stress on several gRPC components; the resolver, the load balancer, and the RPC hotpath. #### long_lived_channel -The client performs a number of large_unary RPCs over a single long-lived +The client performs a number of large_unary RPCs over a single long-lived channel with a fixed but configurable interval between each RPC. ### TODO Tests diff --git a/doc/keepalive.md b/doc/keepalive.md index 4d0271ec181..5080364472a 100644 --- a/doc/keepalive.md +++ b/doc/keepalive.md @@ -14,7 +14,7 @@ The keepalive ping is controlled by two important channel arguments - The above two channel arguments should be sufficient for most users, but the following arguments can also be useful in certain use cases. * **GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS** - * This channel argument if set to 1 (0 : false; 1 : true), allows keepalive pings to be sent even if there are no calls in flight. + * This channel argument if set to 1 (0 : false; 1 : true), allows keepalive pings to be sent even if there are no calls in flight. * **GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA** * This channel argument controls the maximum number of pings that can be sent when there is no other data (data frame or header frame) to be sent. GRPC Core will not continue sending pings if we run over the limit. Setting it to 0 allows sending pings without sending data. * **GRPC_ARG_HTTP2_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS** diff --git a/doc/python/server_reflection.md b/doc/python/server_reflection.md index 8bffdaa5866..5e963ad1f1b 100644 --- a/doc/python/server_reflection.md +++ b/doc/python/server_reflection.md @@ -6,7 +6,7 @@ and more examples how to use server reflection. ## Enable server reflection in Python servers -gRPC Python Server Reflection is an add-on library. To use it, first install +gRPC Python Server Reflection is an add-on library. To use it, first install the [grpcio-reflection] PyPI package into your project. Note that with Python you need to manually register the service @@ -29,7 +29,7 @@ def serve(): server.start() ``` -Please see [greeter_server_with_reflection.py] in the examples directory for the full +Please see [greeter_server_with_reflection.py] in the examples directory for the full example, which extends the gRPC [Python `Greeter` example] on a reflection-enabled server. After starting the server, you can verify that the server reflection diff --git a/doc/security_audit.md b/doc/security_audit.md index 313a9e6b914..a7b935a2d1d 100644 --- a/doc/security_audit.md +++ b/doc/security_audit.md @@ -1,6 +1,6 @@ # gRPC Security Audit -A third-party security audit of gRPC C++ stack was performed by [Cure53](https://cure53.de) in October 2019. The full report can be found [here](https://github.com/grpc/grpc/tree/master/doc/grpc_security_audit.pdf). +A third-party security audit of gRPC C++ stack was performed by [Cure53](https://cure53.de) in October 2019. The full report can be found [here](https://github.com/grpc/grpc/tree/master/doc/grpc_security_audit.pdf). # Addressing grpc_security_audit @@ -21,7 +21,7 @@ Below is a list of alternatives that gRPC team considered. ### Alternative #1: Rewrite gpr_free to take void\*\* -One solution is to change the API of `gpr_free` so that it automatically nulls the given pointer after freeing it. +One solution is to change the API of `gpr_free` so that it automatically nulls the given pointer after freeing it. ``` gpr_free (void** ptr) { @@ -30,7 +30,7 @@ gpr_free (void** ptr) { } ``` -This defensive programming pattern would help protect gRPC from the potential exploits and latent dangling pointer bugs mentioned in the security report. +This defensive programming pattern would help protect gRPC from the potential exploits and latent dangling pointer bugs mentioned in the security report. However, performance would be a significant concern as we are now unconditionally adding a store to every gpr_free call, and there are potentially hundreds of these per RPC. At the RPC layer, this can add up to prohibitive costs. @@ -61,7 +61,7 @@ Because of performance and maintainability concerns, GRP-01-002 will be addresse ## GRP-01-003 Calls to malloc suffer from potential integer overflows The vulnerability, as defined by the report, is that calls to `gpr_malloc` in the C-core codebase may suffer from potential integer overflow in cases where we multiply the array element size by the size of the array. The penetration testers did not identify a concrete place where this occurred, but rather emphasized that the coding pattern itself had potential to lead to vulnerabilities. The report’s suggested solution for GRP-01-003 was to create a `calloc(size_t nmemb, size_t size)` wrapper that contains integer overflow checks. -However, gRPC team firmly believes that gRPC Core should only use integer overflow checks in the places where they’re needed; for example, any place where remote input influences the input to `gpr_malloc` in an unverified way. This is because bounds-checking is very expensive at the RPC layer. +However, gRPC team firmly believes that gRPC Core should only use integer overflow checks in the places where they’re needed; for example, any place where remote input influences the input to `gpr_malloc` in an unverified way. This is because bounds-checking is very expensive at the RPC layer. Determining exactly where bounds-checking is needed requires an audit of tracing each `gpr_malloc` (or `gpr_realloc` or `gpr_zalloc`) call up the stack to determine if the sufficient bounds-checking was performed. This kind of audit, done manually, is fairly expensive engineer-wise. diff --git a/doc/unit_testing.md b/doc/unit_testing.md index ca5648130ce..c83fa5cecf9 100644 --- a/doc/unit_testing.md +++ b/doc/unit_testing.md @@ -75,7 +75,7 @@ grpc_proto_library( ``` -By adding such a flag now a header file `echo_mock.grpc.pb.h` containing the mocked stub will also be generated. +By adding such a flag now a header file `echo_mock.grpc.pb.h` containing the mocked stub will also be generated. This header file can then be included in test files along with a gmock dependency. diff --git a/doc/versioning.md b/doc/versioning.md index 79f573526ee..3d6849c2ce9 100644 --- a/doc/versioning.md +++ b/doc/versioning.md @@ -3,7 +3,7 @@ ## Versioning Overview All gRPC implementations use a three-part version number (`vX.Y.Z`) and follow [semantic versioning](https://semver.org/), which defines the semantics of major, minor and patch components of the version number. In addition to that, gRPC versions evolve according to these rules: -- **Major version bumps** only happen on rare occasions. In order to qualify for a major version bump, certain criteria described later in this document need to be met. Most importantly, a major version increase must not break wire compatibility with other gRPC implementations so that existing gRPC libraries remain fully interoperable. +- **Major version bumps** only happen on rare occasions. In order to qualify for a major version bump, certain criteria described later in this document need to be met. Most importantly, a major version increase must not break wire compatibility with other gRPC implementations so that existing gRPC libraries remain fully interoperable. - **Minor version bumps** happen approx. every 6 weeks as part of the normal release cycle as defined by the gRPC release process. A new release branch named vMAJOR.MINOR.PATCH) is cut every 6 weeks based on the [release schedule](https://github.com/grpc/grpc/blob/master/doc/grpc_release_schedule.md). - **Patch version bump** corresponds to bugfixes done on release branch. @@ -24,7 +24,7 @@ There are also a few extra rules regarding adding new gRPC implementations (e.g. To avoid user confusion and simplify reasoning, the gRPC releases in different languages try to stay synchronized in terms of major and minor version (all languages follow the same release schedule). Nevertheless, because we also strictly follow semantic versioning, there are circumstances in which a gRPC implementation needs to break the version synchronicity and do a major version bump independently of other languages. ### Situations when it's ok to do a major version bump -- **change forced by the language ecosystem:** when the language itself or its standard libraries that we depend on make a breaking change (something which is out of our control), reacting with updating gRPC APIs may be the only adequate response. +- **change forced by the language ecosystem:** when the language itself or its standard libraries that we depend on make a breaking change (something which is out of our control), reacting with updating gRPC APIs may be the only adequate response. - **voluntary change:** Even in non-forced situations, there might be circumstances in which a breaking API change makes sense and represents a net win, but as a rule of thumb breaking changes are very disruptive for users, cause user fragmentation and incur high maintenance costs. Therefore, breaking API changes should be very rare events that need to be considered with extreme care and the bar for accepting such changes is intentionally set very high. Example scenarios where a breaking API change might be adequate: - fixing a security problem which requires changes to API (need to consider the non-breaking alternatives first) From 89cc262dc8f614b9acc73b62c4ba460784df1e66 Mon Sep 17 00:00:00 2001 From: Igor Slepchin Date: Sun, 8 Mar 2020 01:27:04 -0500 Subject: [PATCH 02/80] fix typo --- doc/PROTOCOL-WEB.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/PROTOCOL-WEB.md b/doc/PROTOCOL-WEB.md index 66544543804..1ee93606f65 100644 --- a/doc/PROTOCOL-WEB.md +++ b/doc/PROTOCOL-WEB.md @@ -17,7 +17,7 @@ with the protocol details specified in the For the gRPC-Web protocol, we have decided on the following design goals: * adopt the same framing as “application/grpc” whenever possible -* decouple from HTTP/2 framing which is not, and will never, be directly +* decouple from HTTP/2 framing which is not, and will never be, directly exposed by browsers * support text streams (e.g. base64) in order to provide cross-browser support (e.g. IE-10) From bdb3f860649bb4c136ba36bad3d9363fccbd29fa Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 1 Apr 2020 09:16:02 +0900 Subject: [PATCH 03/80] Fix wrong version in gRPCConfigVersion.cmake and grpc++*.pc They use PACKAGE_VERSION to generate version information but PACKAGE_VERSION may be overridden by find_package(). For example, -DgRPC_CARES_PROVIDER=package with c-ares 1.16.0 overrides PACKAGE_VERSION to 1.16.0. Because c-ares-config-version.cmake has the following line: set(PACKAGE_VERSION "1.16.0") Setting PACKAGE_VERSION in version.cmake is a common CMake convention: https://cmake.org/cmake/help/latest/command/find_package.html#version-selection > PACKAGE_VERSION > > full provided version string So we should use gRPC_CPP_VERSION not PACKAGE_VERSION to generate gRPCConfigVersion.cmake and grpc++*.pc. --- CMakeLists.txt | 6 +++--- templates/CMakeLists.txt.template | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5a52205e480..9593412ce16 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15161,7 +15161,7 @@ include(CMakePackageConfigHelpers) configure_file(cmake/gRPCConfig.cmake.in gRPCConfig.cmake @ONLY) write_basic_package_version_file(${CMAKE_CURRENT_BINARY_DIR}/gRPCConfigVersion.cmake - VERSION ${PACKAGE_VERSION} + VERSION ${gRPC_CPP_VERSION} COMPATIBILITY AnyNewerVersion) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/gRPCConfig.cmake @@ -15228,7 +15228,7 @@ generate_pkgconfig( generate_pkgconfig( "gRPC++" "C++ wrapper for gRPC" - "${PACKAGE_VERSION}" + "${gRPC_CPP_VERSION}" "grpc" "-lgrpc++ -labsl_bad_optional_access -labsl_str_format_internal -labsl_time -labsl_time_zone -labsl_civil_time -labsl_strings -labsl_strings_internal -labsl_throw_delegate -labsl_int128 -labsl_base -labsl_spinlock_wait -labsl_raw_logging_internal -labsl_log_severity -labsl_dynamic_annotations" "" @@ -15238,7 +15238,7 @@ generate_pkgconfig( generate_pkgconfig( "gRPC++ unsecure" "C++ wrapper for gRPC without SSL" - "${PACKAGE_VERSION}" + "${gRPC_CPP_VERSION}" "grpc_unsecure" "-lgrpc++_unsecure -labsl_bad_optional_access -labsl_str_format_internal -labsl_time -labsl_time_zone -labsl_civil_time -labsl_strings -labsl_strings_internal -labsl_throw_delegate -labsl_int128 -labsl_base -labsl_spinlock_wait -labsl_raw_logging_internal -labsl_log_severity -labsl_dynamic_annotations" "" diff --git a/templates/CMakeLists.txt.template b/templates/CMakeLists.txt.template index e6fc6c034ab..40d265ec866 100644 --- a/templates/CMakeLists.txt.template +++ b/templates/CMakeLists.txt.template @@ -673,7 +673,7 @@ configure_file(cmake/gRPCConfig.cmake.in gRPCConfig.cmake @ONLY) write_basic_package_version_file(<%text>${CMAKE_CURRENT_BINARY_DIR}/gRPCConfigVersion.cmake - VERSION <%text>${PACKAGE_VERSION} + VERSION <%text>${gRPC_CPP_VERSION} COMPATIBILITY AnyNewerVersion) install(FILES <%text>${CMAKE_CURRENT_BINARY_DIR}/gRPCConfig.cmake @@ -740,7 +740,7 @@ generate_pkgconfig( "gRPC++" "C++ wrapper for gRPC" - "<%text>${PACKAGE_VERSION}" + "<%text>${gRPC_CPP_VERSION}" "grpc" "${" ".join(("-l" + l) for l in ["grpc++",] + list_absl_lib_files_for("grpc++"))}" "" @@ -750,7 +750,7 @@ generate_pkgconfig( "gRPC++ unsecure" "C++ wrapper for gRPC without SSL" - "<%text>${PACKAGE_VERSION}" + "<%text>${gRPC_CPP_VERSION}" "grpc_unsecure" "${" ".join(("-l" + l) for l in ["grpc++_unsecure",] + list_absl_lib_files_for("grpc++_unsecure"))}" "" From 8ec5886e3938ca2d2d52504137fe857fb7bf5395 Mon Sep 17 00:00:00 2001 From: Misha Efimov Date: Fri, 1 May 2020 10:10:40 -0400 Subject: [PATCH 04/80] Fix spelling error --- src/objective-c/NetworkTransitionBehavior.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/objective-c/NetworkTransitionBehavior.md b/src/objective-c/NetworkTransitionBehavior.md index 51b1e472edc..ef1d3de69b5 100644 --- a/src/objective-c/NetworkTransitionBehavior.md +++ b/src/objective-c/NetworkTransitionBehavior.md @@ -26,7 +26,7 @@ The expected gRPC iOS channel and network transition behaviors are: During the backoff period, any call to the same host will wait until the first of the following events occur: * Connection succeeded; calls will be made using this channel; - * Conncetion failed; calls will be failed and return UNAVAILABLE status code; + * Connection failed; calls will be failed and return UNAVAILABLE status code; * The call's deadline is reached; the call will fail and return DEADLINE\_EXCEEDED status code. The length of backoff period of a channel is reset whenever a connection From 5ea40b7d6cceea3f3d16773592d552f307da6548 Mon Sep 17 00:00:00 2001 From: Tal Kedar Date: Wed, 20 May 2020 12:05:09 -0400 Subject: [PATCH 05/80] Move implementation of ~grpc_server_security_connector() = default to .cc file When the implementation is in the header file, there's an "incomplete type" error caused by grpc_server_credentials. Compiling with clang version 11.0.0 (https://github.com/llvm/llvm-project.git 8697d443ab24d1652d09393efca768cdf4c22b39), Target: x86_64-apple-darwin19.4.0 through cmake's add_subdirectory( lib/grpc ) in a parent project [...] Building CXX object lib/grpc/CMakeFiles/grpc.dir/src/core/lib/security/credentials/tls/grpc_tls_credentials_options.cc.o FAILED: lib/grpc/CMakeFiles/grpc.dir/src/core/lib/security/credentials/tls/grpc_tls_credentials_options.cc.o /usr/local/opt/llvm/latest/bin/clang++ -DCARES_STATICLIB -D__CLANG_SUPPORT_DYN_ANNOTATION__ -I../../lib/grpc/third_party/zlib -I../../lib/grpc/include -I../../lib/grpc -I../../lib/grpc/third_party/address_sorting/include -I../../lib/grpc/third_party/boringssl-with-bazel/src/include -I../../lib/grpc/src/core/ext/upb-generated -I../../lib/grpc/third_party/upb -Ilib/grpc/third_party/zlib -Ilib/grpc/third_party/cares/cares -I../../lib/grpc/third_party/cares/cares -I../../lib/grpc/third_party/abseil-cpp -stdlib=libc++ -nostdinc++ -ibuiltininc -I /usr/local/opt/llvm/latest/include/c++/v1 -fcxx-exceptions -fassociative-math -freciprocal-math -march=native -mtune=native -fforce-emit-vtables -O3 -DNDEBUG -isysroot /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.15.sdk -mmacosx-version-min=10.15 -fPIC -std=c++2a -MD -MT lib/grpc/CMakeFiles/grpc.dir/src/core/lib/security/credentials/tls/grpc_tls_credentials_options.cc.o -MF lib/grpc/CMakeFiles/grpc.dir/src/core/lib/security/credentials/tls/grpc_tls_credentials_options.cc.o.d -o lib/grpc/CMakeFiles/grpc.dir/src/core/lib/security/credentials/tls/grpc_tls_credentials_options.cc.o -c ../../lib/grpc/src/core/lib/security/credentials/tls/grpc_tls_credentials_options.cc In file included from ../../lib/grpc/src/core/lib/security/credentials/tls/grpc_tls_credentials_options.cc:21: In file included from ../../lib/grpc/src/core/lib/security/credentials/tls/grpc_tls_credentials_options.h:28: In file included from ../../lib/grpc/src/core/lib/gprpp/ref_counted.h:36: ../../lib/grpc/src/core/lib/gprpp/ref_counted_ptr.h:103:34: error: member access into incomplete type 'grpc_server_credentials' if (value_ != nullptr) value_->Unref(); ^ ../../lib/grpc/src/core/lib/security/security_connector/security_connector.h:154:3: note: in instantiation of member function 'grpc_core::RefCountedPtr::~RefCountedPtr' requested here ~grpc_server_security_connector() override = default; ^ ../../lib/grpc/include/grpc/grpc_security.h:465:16: note: forward declaration of 'grpc_server_credentials' typedef struct grpc_server_credentials grpc_server_credentials; --- src/core/lib/security/security_connector/security_connector.cc | 2 ++ src/core/lib/security/security_connector/security_connector.h | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/core/lib/security/security_connector/security_connector.cc b/src/core/lib/security/security_connector/security_connector.cc index 3d9b077b633..0400ad6fb8d 100644 --- a/src/core/lib/security/security_connector/security_connector.cc +++ b/src/core/lib/security/security_connector/security_connector.cc @@ -46,6 +46,8 @@ grpc_server_security_connector::grpc_server_security_connector( : grpc_security_connector(url_scheme), server_creds_(std::move(server_creds)) {} +grpc_server_security_connector::~grpc_server_security_connector() = default; + grpc_channel_security_connector::grpc_channel_security_connector( const char* url_scheme, grpc_core::RefCountedPtr channel_creds, diff --git a/src/core/lib/security/security_connector/security_connector.h b/src/core/lib/security/security_connector/security_connector.h index 0f1ac64a44f..2ddd9db7a3c 100644 --- a/src/core/lib/security/security_connector/security_connector.h +++ b/src/core/lib/security/security_connector/security_connector.h @@ -151,7 +151,7 @@ class grpc_server_security_connector : public grpc_security_connector { grpc_server_security_connector( const char* url_scheme, grpc_core::RefCountedPtr server_creds); - ~grpc_server_security_connector() override = default; + ~grpc_server_security_connector() override; virtual void add_handshakers(const grpc_channel_args* args, grpc_pollset_set* interested_parties, From 7324556353e831c57d30973db33df489c3ed3576 Mon Sep 17 00:00:00 2001 From: gkarthiks Date: Sun, 24 May 2020 00:34:35 +0530 Subject: [PATCH 06/80] doc: adding the fourth enum for health check proto docs Signed-off-by: gkarthiks --- doc/health-checking.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/health-checking.md b/doc/health-checking.md index 22b6e1b4c09..0d866e3f4db 100644 --- a/doc/health-checking.md +++ b/doc/health-checking.md @@ -37,6 +37,7 @@ message HealthCheckResponse { UNKNOWN = 0; SERVING = 1; NOT_SERVING = 2; + SERVICE_UNKNOWN = 3; // Used only by the Watch method. } ServingStatus status = 1; } From e04fcd29981186f83d8f9b3c38dd3d496482a2dd Mon Sep 17 00:00:00 2001 From: Mariano Anaya Date: Wed, 22 Apr 2020 16:38:22 +0200 Subject: [PATCH 07/80] [issue-21953] Use the Metadata type In all places where a tuple was used for metadata (in the aio version), replace it by the new ``Metadata`` class. --- src/python/grpcio/grpc/_compression.py | 2 +- .../grpcio/grpc/experimental/aio/_call.py | 10 ++---- .../grpcio/grpc/experimental/aio/_channel.py | 35 ++++++++++++------- .../grpc/experimental/aio/_interceptor.py | 2 +- .../grpcio/grpc/experimental/aio/_typing.py | 9 ++--- .../grpcio_tests/tests_aio/interop/methods.py | 6 ++-- .../grpcio_tests/tests_aio/unit/_common.py | 13 +++---- .../tests_aio/unit/aio_rpc_error_test.py | 7 ++-- .../client_unary_unary_interceptor_test.py | 27 ++++++++------ .../tests_aio/unit/metadata_test.py | 27 ++++++-------- 10 files changed, 76 insertions(+), 62 deletions(-) diff --git a/src/python/grpcio/grpc/_compression.py b/src/python/grpcio/grpc/_compression.py index 45339c3afe2..4035844db97 100644 --- a/src/python/grpcio/grpc/_compression.py +++ b/src/python/grpcio/grpc/_compression.py @@ -39,7 +39,7 @@ def create_channel_option(compression): int(compression)),) if compression else () -def augment_metadata(metadata, compression): +def augment_metadata(metadata, compression) -> tuple: if not metadata and not compression: return None base_metadata = tuple(metadata) if metadata else () diff --git a/src/python/grpcio/grpc/experimental/aio/_call.py b/src/python/grpcio/grpc/experimental/aio/_call.py index a0693921461..35728433987 100644 --- a/src/python/grpcio/grpc/experimental/aio/_call.py +++ b/src/python/grpcio/grpc/experimental/aio/_call.py @@ -28,6 +28,7 @@ from . import _base_call from ._typing import (DeserializingFunction, DoneCallbackType, MetadataType, MetadatumType, RequestIterableType, RequestType, ResponseType, SerializingFunction) +from ._metadata import Metadata __all__ = 'AioRpcError', 'Call', 'UnaryUnaryCall', 'UnaryStreamCall' @@ -58,11 +59,6 @@ class AioRpcError(grpc.RpcError): determined. Hence, its methods no longer needs to be coroutines. """ - # TODO(https://github.com/grpc/grpc/issues/20144) Metadata - # type returned by `initial_metadata` and `trailing_metadata` - # and also taken in the constructor needs to be revisit and make - # it more specific. - _code: grpc.StatusCode _details: Optional[str] _initial_metadata: Optional[MetadataType] @@ -88,8 +84,8 @@ class AioRpcError(grpc.RpcError): super().__init__(self) self._code = code self._details = details - self._initial_metadata = initial_metadata - self._trailing_metadata = trailing_metadata + self._initial_metadata = initial_metadata or Metadata() + self._trailing_metadata = trailing_metadata or Metadata() self._debug_error_string = debug_error_string def code(self) -> grpc.StatusCode: diff --git a/src/python/grpcio/grpc/experimental/aio/_channel.py b/src/python/grpcio/grpc/experimental/aio/_channel.py index 7427872e0b3..3361af477b7 100644 --- a/src/python/grpcio/grpc/experimental/aio/_channel.py +++ b/src/python/grpcio/grpc/experimental/aio/_channel.py @@ -32,8 +32,8 @@ from ._interceptor import ( from ._typing import (ChannelArgumentType, DeserializingFunction, MetadataType, SerializingFunction, RequestIterableType) from ._utils import _timeout_to_deadline +from ._metadata import Metadata -_IMMUTABLE_EMPTY_TUPLE = tuple() _USER_AGENT = 'grpc-python-asyncio/{}'.format(_grpcio_metadata.__version__) if sys.version_info[1] < 7: @@ -88,6 +88,19 @@ class _BaseMultiCallable: self._response_deserializer = response_deserializer self._interceptors = interceptors + @staticmethod + def _init_metadata(metadata: Optional[Metadata] = None, + compression: Optional[grpc.Compression] = None + ) -> Metadata: + """Based on the provided values for or initialise the final + metadata, as it should be used for the current call. + """ + metadata = metadata or Metadata() + if compression: + metadata = Metadata( + *_compression.augment_metadata(metadata, compression)) + return metadata + class UnaryUnaryMultiCallable(_BaseMultiCallable, _base_channel.UnaryUnaryMultiCallable): @@ -96,14 +109,13 @@ class UnaryUnaryMultiCallable(_BaseMultiCallable, request: Any, *, timeout: Optional[float] = None, - metadata: Optional[MetadataType] = _IMMUTABLE_EMPTY_TUPLE, + metadata: Optional[MetadataType] = None, credentials: Optional[grpc.CallCredentials] = None, wait_for_ready: Optional[bool] = None, compression: Optional[grpc.Compression] = None ) -> _base_call.UnaryUnaryCall: - if compression: - metadata = _compression.augment_metadata(metadata, compression) + metadata = self._init_metadata(metadata, compression) if not self._interceptors: call = UnaryUnaryCall(request, _timeout_to_deadline(timeout), metadata, credentials, wait_for_ready, @@ -127,14 +139,13 @@ class UnaryStreamMultiCallable(_BaseMultiCallable, request: Any, *, timeout: Optional[float] = None, - metadata: Optional[MetadataType] = _IMMUTABLE_EMPTY_TUPLE, + metadata: Optional[MetadataType] = None, credentials: Optional[grpc.CallCredentials] = None, wait_for_ready: Optional[bool] = None, compression: Optional[grpc.Compression] = None ) -> _base_call.UnaryStreamCall: - if compression: - metadata = _compression.augment_metadata(metadata, compression) + metadata = self._init_metadata(metadata, compression) deadline = _timeout_to_deadline(timeout) if not self._interceptors: @@ -158,14 +169,13 @@ class StreamUnaryMultiCallable(_BaseMultiCallable, def __call__(self, request_iterator: Optional[RequestIterableType] = None, timeout: Optional[float] = None, - metadata: Optional[MetadataType] = _IMMUTABLE_EMPTY_TUPLE, + metadata: Optional[MetadataType] = None, credentials: Optional[grpc.CallCredentials] = None, wait_for_ready: Optional[bool] = None, compression: Optional[grpc.Compression] = None ) -> _base_call.StreamUnaryCall: - if compression: - metadata = _compression.augment_metadata(metadata, compression) + metadata = self._init_metadata(metadata, compression) deadline = _timeout_to_deadline(timeout) if not self._interceptors: @@ -189,14 +199,13 @@ class StreamStreamMultiCallable(_BaseMultiCallable, def __call__(self, request_iterator: Optional[RequestIterableType] = None, timeout: Optional[float] = None, - metadata: Optional[MetadataType] = _IMMUTABLE_EMPTY_TUPLE, + metadata: Optional[MetadataType] = None, credentials: Optional[grpc.CallCredentials] = None, wait_for_ready: Optional[bool] = None, compression: Optional[grpc.Compression] = None ) -> _base_call.StreamStreamCall: - if compression: - metadata = _compression.augment_metadata(metadata, compression) + metadata = self._init_metadata(metadata, compression) deadline = _timeout_to_deadline(timeout) if not self._interceptors: diff --git a/src/python/grpcio/grpc/experimental/aio/_interceptor.py b/src/python/grpcio/grpc/experimental/aio/_interceptor.py index e276ae0c922..8a28a61c8ba 100644 --- a/src/python/grpcio/grpc/experimental/aio/_interceptor.py +++ b/src/python/grpcio/grpc/experimental/aio/_interceptor.py @@ -248,7 +248,7 @@ class StreamStreamClientInterceptor(ClientInterceptor, metaclass=ABCMeta): class InterceptedCall: - """Base implementation for all intecepted call arities. + """Base implementation for all intercepted call arities. Interceptors might have some work to do before the RPC invocation with the capacity of changing the invocation parameters, and some work to do diff --git a/src/python/grpcio/grpc/experimental/aio/_typing.py b/src/python/grpcio/grpc/experimental/aio/_typing.py index a02ec8ff803..7e2e8da8a06 100644 --- a/src/python/grpcio/grpc/experimental/aio/_typing.py +++ b/src/python/grpcio/grpc/experimental/aio/_typing.py @@ -13,17 +13,18 @@ # limitations under the License. """Common types for gRPC Async API""" -from typing import (Any, AnyStr, AsyncIterable, Callable, Iterable, Sequence, - Tuple, TypeVar, Union) +from typing import (Any, AsyncIterable, Callable, Iterable, Sequence, Tuple, + TypeVar, Union) from grpc._cython.cygrpc import EOF +from ._metadata import Metadata, MetadataKey, MetadataValue RequestType = TypeVar('RequestType') ResponseType = TypeVar('ResponseType') SerializingFunction = Callable[[Any], bytes] DeserializingFunction = Callable[[bytes], Any] -MetadatumType = Tuple[str, AnyStr] -MetadataType = Sequence[MetadatumType] +MetadatumType = Tuple[MetadataKey, MetadataValue] +MetadataType = Metadata ChannelArgumentType = Sequence[Tuple[str, Any]] EOFType = type(EOF) DoneCallbackType = Callable[[Any], None] diff --git a/src/python/grpcio_tests/tests_aio/interop/methods.py b/src/python/grpcio_tests/tests_aio/interop/methods.py index 019b3bca894..4ff10ff1572 100644 --- a/src/python/grpcio_tests/tests_aio/interop/methods.py +++ b/src/python/grpcio_tests/tests_aio/interop/methods.py @@ -287,8 +287,10 @@ async def _unimplemented_service(stub: test_pb2_grpc.UnimplementedServiceStub): async def _custom_metadata(stub: test_pb2_grpc.TestServiceStub): initial_metadata_value = "test_initial_metadata_value" trailing_metadata_value = b"\x0a\x0b\x0a\x0b\x0a\x0b" - metadata = ((_INITIAL_METADATA_KEY, initial_metadata_value), - (_TRAILING_METADATA_KEY, trailing_metadata_value)) + metadata = aio.Metadata( + (_INITIAL_METADATA_KEY, initial_metadata_value), + (_TRAILING_METADATA_KEY, trailing_metadata_value), + ) async def _validate_metadata(call): initial_metadata = dict(await call.initial_metadata()) diff --git a/src/python/grpcio_tests/tests_aio/unit/_common.py b/src/python/grpcio_tests/tests_aio/unit/_common.py index dab9454c58d..7f9e2e89701 100644 --- a/src/python/grpcio_tests/tests_aio/unit/_common.py +++ b/src/python/grpcio_tests/tests_aio/unit/_common.py @@ -16,18 +16,19 @@ import asyncio import grpc from typing import AsyncIterable from grpc.experimental import aio -from grpc.experimental.aio._typing import MetadataType, MetadatumType +from grpc.experimental.aio._typing import MetadataType, MetadatumType, MetadataKey, MetadataValue from tests.unit.framework.common import test_constants def seen_metadata(expected: MetadataType, actual: MetadataType): - return not bool(set(expected) - set(actual)) + return not bool(set(tuple(expected)) - set(tuple(actual))) -def seen_metadatum(expected: MetadatumType, actual: MetadataType): - metadata_dict = dict(actual) - return metadata_dict.get(expected[0]) == expected[1] +def seen_metadatum(expected_key: MetadataKey, expected_value: MetadataValue, + actual: MetadataType) -> bool: + obtained = actual[expected_key] + assert obtained == expected_value async def block_until_certain_state(channel: aio.Channel, @@ -50,7 +51,7 @@ def inject_callbacks(call: aio.Call): second_callback_ran = asyncio.Event() def second_callback(call): - # Validate that all resopnses have been received + # Validate that all responses have been received # and the call is an end state. assert call.done() second_callback_ran.set() diff --git a/src/python/grpcio_tests/tests_aio/unit/aio_rpc_error_test.py b/src/python/grpcio_tests/tests_aio/unit/aio_rpc_error_test.py index fbcd99b39e3..9bd652a43a6 100644 --- a/src/python/grpcio_tests/tests_aio/unit/aio_rpc_error_test.py +++ b/src/python/grpcio_tests/tests_aio/unit/aio_rpc_error_test.py @@ -18,11 +18,14 @@ import unittest import grpc +from grpc.experimental import aio from grpc.experimental.aio._call import AioRpcError from tests_aio.unit._test_base import AioTestBase -_TEST_INITIAL_METADATA = ('initial metadata',) -_TEST_TRAILING_METADATA = ('trailing metadata',) +_TEST_INITIAL_METADATA = aio.Metadata( + ('initial metadata key', 'initial metadata value')) +_TEST_TRAILING_METADATA = aio.Metadata( + ('trailing metadata key', 'trailing metadata value')) _TEST_DEBUG_ERROR_STRING = '{This is a debug string}' diff --git a/src/python/grpcio_tests/tests_aio/unit/client_unary_unary_interceptor_test.py b/src/python/grpcio_tests/tests_aio/unit/client_unary_unary_interceptor_test.py index 8f5a356ca4a..ae1ad54acd9 100644 --- a/src/python/grpcio_tests/tests_aio/unit/client_unary_unary_interceptor_test.py +++ b/src/python/grpcio_tests/tests_aio/unit/client_unary_unary_interceptor_test.py @@ -25,7 +25,7 @@ from tests_aio.unit._test_base import AioTestBase from src.proto.grpc.testing import messages_pb2, test_pb2_grpc _LOCAL_CANCEL_DETAILS_EXPECTATION = 'Locally cancelled by application!' -_INITIAL_METADATA_TO_INJECT = ( +_INITIAL_METADATA_TO_INJECT = aio.Metadata( (_INITIAL_METADATA_KEY, 'extra info'), (_TRAILING_METADATA_KEY, b'\x13\x37'), ) @@ -162,7 +162,7 @@ class TestUnaryUnaryClientInterceptor(AioTestBase): async def test_retry(self): class RetryInterceptor(aio.UnaryUnaryClientInterceptor): - """Simulates a Retry Interceptor which ends up by making + """Simulates a Retry Interceptor which ends up by making two RPC calls.""" def __init__(self): @@ -550,11 +550,12 @@ class TestInterceptedUnaryUnaryCall(AioTestBase): async def intercept_unary_unary(self, continuation, client_call_details, request): + new_metadata = aio.Metadata(*client_call_details.metadata, + *_INITIAL_METADATA_TO_INJECT) new_details = aio.ClientCallDetails( method=client_call_details.method, timeout=client_call_details.timeout, - metadata=client_call_details.metadata + - _INITIAL_METADATA_TO_INJECT, + metadata=new_metadata, credentials=client_call_details.credentials, wait_for_ready=client_call_details.wait_for_ready, ) @@ -568,14 +569,20 @@ class TestInterceptedUnaryUnaryCall(AioTestBase): # Expected to see the echoed initial metadata self.assertTrue( - _common.seen_metadatum(_INITIAL_METADATA_TO_INJECT[0], await - call.initial_metadata())) - + _common.seen_metadatum( + expected_key=_INITIAL_METADATA_KEY, + expected_value=_INITIAL_METADATA_TO_INJECT[ + _INITIAL_METADATA_KEY], + actual=await call.initial_metadata(), + )) # Expected to see the echoed trailing metadata self.assertTrue( - _common.seen_metadatum(_INITIAL_METADATA_TO_INJECT[1], await - call.trailing_metadata())) - + _common.seen_metadatum( + expected_key=_TRAILING_METADATA_KEY, + expected_value=_INITIAL_METADATA_TO_INJECT[ + _TRAILING_METADATA_KEY], + actual=await call.trailing_metadata(), + )) self.assertEqual(await call.code(), grpc.StatusCode.OK) async def test_add_done_callback_before_finishes(self): diff --git a/src/python/grpcio_tests/tests_aio/unit/metadata_test.py b/src/python/grpcio_tests/tests_aio/unit/metadata_test.py index 6551e4ca084..16f91430fce 100644 --- a/src/python/grpcio_tests/tests_aio/unit/metadata_test.py +++ b/src/python/grpcio_tests/tests_aio/unit/metadata_test.py @@ -37,38 +37,33 @@ _TEST_STREAM_STREAM = '/test/TestStreamStream' _REQUEST = b'\x00\x00\x00' _RESPONSE = b'\x01\x01\x01' -_INITIAL_METADATA_FROM_CLIENT_TO_SERVER = ( +_INITIAL_METADATA_FROM_CLIENT_TO_SERVER = aio.Metadata( ('client-to-server', 'question'), ('client-to-server-bin', b'\x07\x07\x07'), ) -_INITIAL_METADATA_FROM_SERVER_TO_CLIENT = ( +_INITIAL_METADATA_FROM_SERVER_TO_CLIENT = aio.Metadata( ('server-to-client', 'answer'), ('server-to-client-bin', b'\x06\x06\x06'), ) -_TRAILING_METADATA = (('a-trailing-metadata', 'stack-trace'), - ('a-trailing-metadata-bin', b'\x05\x05\x05')) -_INITIAL_METADATA_FOR_GENERIC_HANDLER = (('a-must-have-key', 'secret'),) +_TRAILING_METADATA = aio.Metadata( + ('a-trailing-metadata', 'stack-trace'), + ('a-trailing-metadata-bin', b'\x05\x05\x05'), +) +_INITIAL_METADATA_FOR_GENERIC_HANDLER = aio.Metadata( + ('a-must-have-key', 'secret'),) _INVALID_METADATA_TEST_CASES = ( ( TypeError, - ((42, 42),), - ), - ( - TypeError, - (({}, {}),), - ), - ( - TypeError, - (('normal', object()),), + aio.Metadata((42, 42),), ), ( TypeError, - object(), + aio.Metadata(({}, {}),), ), ( TypeError, - (object(),), + aio.Metadata(('normal', object()),), ), ) From e9dadf46bfba716c7c88ce4f69cb10b7b1844518 Mon Sep 17 00:00:00 2001 From: Mariano Anaya Date: Tue, 26 May 2020 17:00:13 +0200 Subject: [PATCH 08/80] [issue-24953] Fix tests, format, & types Fixes https://github.com/grpc/grpc/issues/21953 --- .../grpcio/grpc/experimental/aio/_call.py | 13 +++++---- .../grpcio/grpc/experimental/aio/_channel.py | 2 +- .../grpcio_tests/tests_aio/interop/methods.py | 5 ++-- .../grpcio_tests/tests_aio/unit/_common.py | 2 +- .../grpcio_tests/tests_aio/unit/call_test.py | 10 +++---- .../client_stream_unary_interceptor_test.py | 12 ++++----- .../client_unary_stream_interceptor_test.py | 4 +-- .../client_unary_unary_interceptor_test.py | 22 ++++++++------- .../tests_aio/unit/compatibility_test.py | 3 ++- .../tests_aio/unit/metadata_test.py | 27 ++++++++++--------- .../tests_aio/unit/server_interceptor_test.py | 4 +-- 11 files changed, 56 insertions(+), 48 deletions(-) diff --git a/src/python/grpcio/grpc/experimental/aio/_call.py b/src/python/grpcio/grpc/experimental/aio/_call.py index 35728433987..7ecc2dd8e1b 100644 --- a/src/python/grpcio/grpc/experimental/aio/_call.py +++ b/src/python/grpcio/grpc/experimental/aio/_call.py @@ -25,10 +25,10 @@ from grpc import _common from grpc._cython import cygrpc from . import _base_call +from ._metadata import Metadata from ._typing import (DeserializingFunction, DoneCallbackType, MetadataType, MetadatumType, RequestIterableType, RequestType, ResponseType, SerializingFunction) -from ._metadata import Metadata __all__ = 'AioRpcError', 'Call', 'UnaryUnaryCall', 'UnaryStreamCall' @@ -84,8 +84,8 @@ class AioRpcError(grpc.RpcError): super().__init__(self) self._code = code self._details = details - self._initial_metadata = initial_metadata or Metadata() - self._trailing_metadata = trailing_metadata or Metadata() + self._initial_metadata = Metadata(*(initial_metadata or ())) + self._trailing_metadata = Metadata(*(trailing_metadata or ())) self._debug_error_string = debug_error_string def code(self) -> grpc.StatusCode: @@ -205,10 +205,13 @@ class Call: return self._cython_call.time_remaining() async def initial_metadata(self) -> MetadataType: - return await self._cython_call.initial_metadata() + raw_metadata_tuple = await self._cython_call.initial_metadata() + return Metadata(*(raw_metadata_tuple or ())) async def trailing_metadata(self) -> MetadataType: - return (await self._cython_call.status()).trailing_metadata() + raw_metadata_tuple = (await + self._cython_call.status()).trailing_metadata() + return Metadata(*(raw_metadata_tuple or ())) async def code(self) -> grpc.StatusCode: cygrpc_code = (await self._cython_call.status()).code() diff --git a/src/python/grpcio/grpc/experimental/aio/_channel.py b/src/python/grpcio/grpc/experimental/aio/_channel.py index 3361af477b7..3ac12bf6139 100644 --- a/src/python/grpcio/grpc/experimental/aio/_channel.py +++ b/src/python/grpcio/grpc/experimental/aio/_channel.py @@ -29,10 +29,10 @@ from ._interceptor import ( InterceptedStreamUnaryCall, InterceptedStreamStreamCall, ClientInterceptor, UnaryUnaryClientInterceptor, UnaryStreamClientInterceptor, StreamUnaryClientInterceptor, StreamStreamClientInterceptor) +from ._metadata import Metadata from ._typing import (ChannelArgumentType, DeserializingFunction, MetadataType, SerializingFunction, RequestIterableType) from ._utils import _timeout_to_deadline -from ._metadata import Metadata _USER_AGENT = 'grpc-python-asyncio/{}'.format(_grpcio_metadata.__version__) diff --git a/src/python/grpcio_tests/tests_aio/interop/methods.py b/src/python/grpcio_tests/tests_aio/interop/methods.py index 4ff10ff1572..706f4249be3 100644 --- a/src/python/grpcio_tests/tests_aio/interop/methods.py +++ b/src/python/grpcio_tests/tests_aio/interop/methods.py @@ -293,12 +293,13 @@ async def _custom_metadata(stub: test_pb2_grpc.TestServiceStub): ) async def _validate_metadata(call): - initial_metadata = dict(await call.initial_metadata()) + initial_metadata = await call.initial_metadata() if initial_metadata[_INITIAL_METADATA_KEY] != initial_metadata_value: raise ValueError('expected initial metadata %s, got %s' % (initial_metadata_value, initial_metadata[_INITIAL_METADATA_KEY])) - trailing_metadata = dict(await call.trailing_metadata()) + + trailing_metadata = await call.trailing_metadata() if trailing_metadata[_TRAILING_METADATA_KEY] != trailing_metadata_value: raise ValueError('expected trailing metadata %s, got %s' % (trailing_metadata_value, diff --git a/src/python/grpcio_tests/tests_aio/unit/_common.py b/src/python/grpcio_tests/tests_aio/unit/_common.py index 7f9e2e89701..a4a9236069c 100644 --- a/src/python/grpcio_tests/tests_aio/unit/_common.py +++ b/src/python/grpcio_tests/tests_aio/unit/_common.py @@ -28,7 +28,7 @@ def seen_metadata(expected: MetadataType, actual: MetadataType): def seen_metadatum(expected_key: MetadataKey, expected_value: MetadataValue, actual: MetadataType) -> bool: obtained = actual[expected_key] - assert obtained == expected_value + return obtained == expected_value async def block_until_certain_state(channel: aio.Channel, diff --git a/src/python/grpcio_tests/tests_aio/unit/call_test.py b/src/python/grpcio_tests/tests_aio/unit/call_test.py index 3ce2a2f7b52..94a36a4c070 100644 --- a/src/python/grpcio_tests/tests_aio/unit/call_test.py +++ b/src/python/grpcio_tests/tests_aio/unit/call_test.py @@ -102,11 +102,11 @@ class TestUnaryUnaryCall(_MulticallableTestMixin, AioTestBase): async def test_call_initial_metadata_awaitable(self): call = self._stub.UnaryCall(messages_pb2.SimpleRequest()) - self.assertEqual((), await call.initial_metadata()) + self.assertEqual(await call.initial_metadata(), aio.Metadata()) async def test_call_trailing_metadata_awaitable(self): call = self._stub.UnaryCall(messages_pb2.SimpleRequest()) - self.assertEqual((), await call.trailing_metadata()) + self.assertEqual(await call.trailing_metadata(), aio.Metadata()) async def test_call_initial_metadata_cancelable(self): coro_started = asyncio.Event() @@ -122,7 +122,7 @@ class TestUnaryUnaryCall(_MulticallableTestMixin, AioTestBase): # Test that initial metadata can still be asked thought # a cancellation happened with the previous task - self.assertEqual((), await call.initial_metadata()) + self.assertEqual(await call.initial_metadata(), aio.Metadata()) async def test_call_initial_metadata_multiple_waiters(self): call = self._stub.UnaryCall(messages_pb2.SimpleRequest()) @@ -134,8 +134,8 @@ class TestUnaryUnaryCall(_MulticallableTestMixin, AioTestBase): task2 = self.loop.create_task(coro()) await call - - self.assertEqual([(), ()], await asyncio.gather(*[task1, task2])) + expected = [aio.Metadata() for _ in range(2)] + self.assertEqual(await asyncio.gather(*[task1, task2]), expected) async def test_call_code_cancelable(self): coro_started = asyncio.Event() diff --git a/src/python/grpcio_tests/tests_aio/unit/client_stream_unary_interceptor_test.py b/src/python/grpcio_tests/tests_aio/unit/client_stream_unary_interceptor_test.py index 2d58cff0e41..b9a04af00dc 100644 --- a/src/python/grpcio_tests/tests_aio/unit/client_stream_unary_interceptor_test.py +++ b/src/python/grpcio_tests/tests_aio/unit/client_stream_unary_interceptor_test.py @@ -92,8 +92,8 @@ class TestStreamUnaryClientInterceptor(AioTestBase): self.assertEqual(_NUM_STREAM_REQUESTS * _REQUEST_PAYLOAD_SIZE, response.aggregated_payload_size) self.assertEqual(await call.code(), grpc.StatusCode.OK) - self.assertEqual(await call.initial_metadata(), ()) - self.assertEqual(await call.trailing_metadata(), ()) + self.assertEqual(await call.initial_metadata(), aio.Metadata()) + self.assertEqual(await call.trailing_metadata(), aio.Metadata()) self.assertEqual(await call.details(), '') self.assertEqual(await call.debug_error_string(), '') self.assertEqual(call.cancel(), False) @@ -131,8 +131,8 @@ class TestStreamUnaryClientInterceptor(AioTestBase): self.assertEqual(_NUM_STREAM_REQUESTS * _REQUEST_PAYLOAD_SIZE, response.aggregated_payload_size) self.assertEqual(await call.code(), grpc.StatusCode.OK) - self.assertEqual(await call.initial_metadata(), ()) - self.assertEqual(await call.trailing_metadata(), ()) + self.assertEqual(await call.initial_metadata(), aio.Metadata()) + self.assertEqual(await call.trailing_metadata(), aio.Metadata()) self.assertEqual(await call.details(), '') self.assertEqual(await call.debug_error_string(), '') self.assertEqual(call.cancel(), False) @@ -230,8 +230,8 @@ class TestStreamUnaryClientInterceptor(AioTestBase): self.assertEqual(_NUM_STREAM_REQUESTS * _REQUEST_PAYLOAD_SIZE, response.aggregated_payload_size) self.assertEqual(await call.code(), grpc.StatusCode.OK) - self.assertEqual(await call.initial_metadata(), ()) - self.assertEqual(await call.trailing_metadata(), ()) + self.assertEqual(await call.initial_metadata(), aio.Metadata()) + self.assertEqual(await call.trailing_metadata(), aio.Metadata()) self.assertEqual(await call.details(), '') self.assertEqual(await call.debug_error_string(), '') self.assertEqual(call.cancel(), False) diff --git a/src/python/grpcio_tests/tests_aio/unit/client_unary_stream_interceptor_test.py b/src/python/grpcio_tests/tests_aio/unit/client_unary_stream_interceptor_test.py index 6137538ffca..fd542fd16e9 100644 --- a/src/python/grpcio_tests/tests_aio/unit/client_unary_stream_interceptor_test.py +++ b/src/python/grpcio_tests/tests_aio/unit/client_unary_stream_interceptor_test.py @@ -96,8 +96,8 @@ class TestUnaryStreamClientInterceptor(AioTestBase): self.assertEqual(response_cnt, _NUM_STREAM_RESPONSES) self.assertEqual(await call.code(), grpc.StatusCode.OK) - self.assertEqual(await call.initial_metadata(), ()) - self.assertEqual(await call.trailing_metadata(), ()) + self.assertEqual(await call.initial_metadata(), aio.Metadata()) + self.assertEqual(await call.trailing_metadata(), aio.Metadata()) self.assertEqual(await call.details(), '') self.assertEqual(await call.debug_error_string(), '') self.assertEqual(call.cancel(), False) diff --git a/src/python/grpcio_tests/tests_aio/unit/client_unary_unary_interceptor_test.py b/src/python/grpcio_tests/tests_aio/unit/client_unary_unary_interceptor_test.py index ae1ad54acd9..e64daec7df4 100644 --- a/src/python/grpcio_tests/tests_aio/unit/client_unary_unary_interceptor_test.py +++ b/src/python/grpcio_tests/tests_aio/unit/client_unary_unary_interceptor_test.py @@ -302,8 +302,8 @@ class TestInterceptedUnaryUnaryCall(AioTestBase): self.assertEqual(type(response), messages_pb2.SimpleResponse) self.assertEqual(await call.code(), grpc.StatusCode.OK) self.assertEqual(await call.details(), '') - self.assertEqual(await call.initial_metadata(), ()) - self.assertEqual(await call.trailing_metadata(), ()) + self.assertEqual(await call.initial_metadata(), aio.Metadata()) + self.assertEqual(await call.trailing_metadata(), aio.Metadata()) async def test_call_ok_awaited(self): @@ -331,8 +331,8 @@ class TestInterceptedUnaryUnaryCall(AioTestBase): self.assertEqual(type(response), messages_pb2.SimpleResponse) self.assertEqual(await call.code(), grpc.StatusCode.OK) self.assertEqual(await call.details(), '') - self.assertEqual(await call.initial_metadata(), ()) - self.assertEqual(await call.trailing_metadata(), ()) + self.assertEqual(await call.initial_metadata(), aio.Metadata()) + self.assertEqual(await call.trailing_metadata(), aio.Metadata()) async def test_call_rpc_error(self): @@ -364,8 +364,8 @@ class TestInterceptedUnaryUnaryCall(AioTestBase): self.assertEqual(await call.code(), grpc.StatusCode.DEADLINE_EXCEEDED) self.assertEqual(await call.details(), 'Deadline Exceeded') - self.assertEqual(await call.initial_metadata(), ()) - self.assertEqual(await call.trailing_metadata(), ()) + self.assertEqual(await call.initial_metadata(), aio.Metadata()) + self.assertEqual(await call.trailing_metadata(), aio.Metadata()) async def test_call_rpc_error_awaited(self): @@ -398,8 +398,8 @@ class TestInterceptedUnaryUnaryCall(AioTestBase): self.assertEqual(await call.code(), grpc.StatusCode.DEADLINE_EXCEEDED) self.assertEqual(await call.details(), 'Deadline Exceeded') - self.assertEqual(await call.initial_metadata(), ()) - self.assertEqual(await call.trailing_metadata(), ()) + self.assertEqual(await call.initial_metadata(), aio.Metadata()) + self.assertEqual(await call.trailing_metadata(), aio.Metadata()) async def test_cancel_before_rpc(self): @@ -541,8 +541,10 @@ class TestInterceptedUnaryUnaryCall(AioTestBase): self.assertEqual(await call.code(), grpc.StatusCode.CANCELLED) self.assertEqual(await call.details(), _LOCAL_CANCEL_DETAILS_EXPECTATION) - self.assertEqual(await call.initial_metadata(), tuple()) - self.assertEqual(await call.trailing_metadata(), None) + self.assertEqual(await call.initial_metadata(), aio.Metadata()) + self.assertEqual( + await call.trailing_metadata(), aio.Metadata(), + "When the raw response is None, empty metadata is returned") async def test_initial_metadata_modification(self): diff --git a/src/python/grpcio_tests/tests_aio/unit/compatibility_test.py b/src/python/grpcio_tests/tests_aio/unit/compatibility_test.py index 1e6e3598b8b..0bb3a3acc89 100644 --- a/src/python/grpcio_tests/tests_aio/unit/compatibility_test.py +++ b/src/python/grpcio_tests/tests_aio/unit/compatibility_test.py @@ -255,7 +255,8 @@ class TestCompatibility(AioTestBase): self._adhoc_handlers.set_adhoc_handler(metadata_unary_unary) call = self._async_channel.unary_unary(_ADHOC_METHOD)(_REQUEST) self.assertTrue( - _common.seen_metadata(metadata, await call.initial_metadata())) + _common.seen_metadata(aio.Metadata(*metadata), await + call.initial_metadata())) async def test_sync_unary_unary_abort(self): diff --git a/src/python/grpcio_tests/tests_aio/unit/metadata_test.py b/src/python/grpcio_tests/tests_aio/unit/metadata_test.py index 16f91430fce..59f5596c70b 100644 --- a/src/python/grpcio_tests/tests_aio/unit/metadata_test.py +++ b/src/python/grpcio_tests/tests_aio/unit/metadata_test.py @@ -55,15 +55,15 @@ _INITIAL_METADATA_FOR_GENERIC_HANDLER = aio.Metadata( _INVALID_METADATA_TEST_CASES = ( ( TypeError, - aio.Metadata((42, 42),), + ((42, 42),), ), ( TypeError, - aio.Metadata(({}, {}),), + ((None, {}),), ), ( TypeError, - aio.Metadata(('normal', object()),), + (('normal', object()),), ), ) @@ -100,13 +100,13 @@ class _TestGenericHandlerForMethods(grpc.GenericRpcHandler): async def _test_server_to_client(request, context): assert _REQUEST == request await context.send_initial_metadata( - _INITIAL_METADATA_FROM_SERVER_TO_CLIENT) + tuple(_INITIAL_METADATA_FROM_SERVER_TO_CLIENT)) return _RESPONSE @staticmethod async def _test_trailing_metadata(request, context): assert _REQUEST == request - context.set_trailing_metadata(_TRAILING_METADATA) + context.set_trailing_metadata(tuple(_TRAILING_METADATA)) return _RESPONSE @staticmethod @@ -115,21 +115,21 @@ class _TestGenericHandlerForMethods(grpc.GenericRpcHandler): assert _common.seen_metadata(_INITIAL_METADATA_FROM_CLIENT_TO_SERVER, context.invocation_metadata()) await context.send_initial_metadata( - _INITIAL_METADATA_FROM_SERVER_TO_CLIENT) + tuple(_INITIAL_METADATA_FROM_SERVER_TO_CLIENT)) yield _RESPONSE - context.set_trailing_metadata(_TRAILING_METADATA) + context.set_trailing_metadata(tuple(_TRAILING_METADATA)) @staticmethod async def _test_stream_unary(request_iterator, context): assert _common.seen_metadata(_INITIAL_METADATA_FROM_CLIENT_TO_SERVER, context.invocation_metadata()) await context.send_initial_metadata( - _INITIAL_METADATA_FROM_SERVER_TO_CLIENT) + tuple(_INITIAL_METADATA_FROM_SERVER_TO_CLIENT)) async for request in request_iterator: assert _REQUEST == request - context.set_trailing_metadata(_TRAILING_METADATA) + context.set_trailing_metadata(tuple(_TRAILING_METADATA)) return _RESPONSE @staticmethod @@ -137,13 +137,13 @@ class _TestGenericHandlerForMethods(grpc.GenericRpcHandler): assert _common.seen_metadata(_INITIAL_METADATA_FROM_CLIENT_TO_SERVER, context.invocation_metadata()) await context.send_initial_metadata( - _INITIAL_METADATA_FROM_SERVER_TO_CLIENT) + tuple(_INITIAL_METADATA_FROM_SERVER_TO_CLIENT)) async for request in request_iterator: assert _REQUEST == request yield _RESPONSE - context.set_trailing_metadata(_TRAILING_METADATA) + context.set_trailing_metadata(tuple(_TRAILING_METADATA)) def service(self, handler_call_details): return self._routing_table.get(handler_call_details.method) @@ -193,6 +193,7 @@ class TestMetadata(AioTestBase): async def test_from_server_to_client(self): multicallable = self._client.unary_unary(_TEST_SERVER_TO_CLIENT) call = multicallable(_REQUEST) + self.assertEqual(_INITIAL_METADATA_FROM_SERVER_TO_CLIENT, await call.initial_metadata()) self.assertEqual(_RESPONSE, await call) @@ -207,8 +208,8 @@ class TestMetadata(AioTestBase): async def test_from_client_to_server_with_list(self): multicallable = self._client.unary_unary(_TEST_CLIENT_TO_SERVER) - call = multicallable( - _REQUEST, metadata=list(_INITIAL_METADATA_FROM_CLIENT_TO_SERVER)) + call = multicallable(_REQUEST, + metadata=_INITIAL_METADATA_FROM_CLIENT_TO_SERVER) self.assertEqual(_RESPONSE, await call) self.assertEqual(grpc.StatusCode.OK, await call.code()) diff --git a/src/python/grpcio_tests/tests_aio/unit/server_interceptor_test.py b/src/python/grpcio_tests/tests_aio/unit/server_interceptor_test.py index f85e46c379a..d891ecdb771 100644 --- a/src/python/grpcio_tests/tests_aio/unit/server_interceptor_test.py +++ b/src/python/grpcio_tests/tests_aio/unit/server_interceptor_test.py @@ -198,7 +198,7 @@ class TestServerInterceptor(AioTestBase): request_serializer=messages_pb2.SimpleRequest.SerializeToString, response_deserializer=messages_pb2.SimpleResponse.FromString) - metadata = (('key', 'value'),) + metadata = aio.Metadata(('key', 'value'),) call = multicallable(messages_pb2.SimpleRequest(), metadata=metadata) await call @@ -208,7 +208,7 @@ class TestServerInterceptor(AioTestBase): ], record) record.clear() - metadata = (('key', 'value'), ('secret', '42')) + metadata = aio.Metadata(('key', 'value'), ('secret', '42')) call = multicallable(messages_pb2.SimpleRequest(), metadata=metadata) await call From 8fcc77a3109080d41d9a62763a17cde18aaa90d1 Mon Sep 17 00:00:00 2001 From: Mariano Anaya Date: Thu, 28 May 2020 10:41:38 +0200 Subject: [PATCH 09/80] [issue-21953] Improvements from review * Replace ``MetadataType`` by ``Metadata`` in all places * Fix annotations * Use the new ``Metadata.from_tuple`` to create Metadata objects --- src/python/grpcio/grpc/_compression.py | 2 +- .../grpcio/grpc/experimental/aio/_call.py | 42 +++++++++---------- .../grpcio/grpc/experimental/aio/_metadata.py | 10 ++++- .../tests_aio/unit/_metadata_test.py | 12 ++++++ .../grpcio_tests/tests_aio/unit/call_test.py | 8 ++-- .../tests_aio/unit/metadata_test.py | 4 ++ 6 files changed, 50 insertions(+), 28 deletions(-) diff --git a/src/python/grpcio/grpc/_compression.py b/src/python/grpcio/grpc/_compression.py index 4035844db97..45339c3afe2 100644 --- a/src/python/grpcio/grpc/_compression.py +++ b/src/python/grpcio/grpc/_compression.py @@ -39,7 +39,7 @@ def create_channel_option(compression): int(compression)),) if compression else () -def augment_metadata(metadata, compression) -> tuple: +def augment_metadata(metadata, compression): if not metadata and not compression: return None base_metadata = tuple(metadata) if metadata else () diff --git a/src/python/grpcio/grpc/experimental/aio/_call.py b/src/python/grpcio/grpc/experimental/aio/_call.py index 7ecc2dd8e1b..c121bd6b76d 100644 --- a/src/python/grpcio/grpc/experimental/aio/_call.py +++ b/src/python/grpcio/grpc/experimental/aio/_call.py @@ -26,7 +26,7 @@ from grpc._cython import cygrpc from . import _base_call from ._metadata import Metadata -from ._typing import (DeserializingFunction, DoneCallbackType, MetadataType, +from ._typing import (DeserializingFunction, DoneCallbackType, MetadatumType, RequestIterableType, RequestType, ResponseType, SerializingFunction) @@ -61,15 +61,15 @@ class AioRpcError(grpc.RpcError): _code: grpc.StatusCode _details: Optional[str] - _initial_metadata: Optional[MetadataType] - _trailing_metadata: Optional[MetadataType] + _initial_metadata: Optional[Metadata] + _trailing_metadata: Optional[Metadata] _debug_error_string: Optional[str] def __init__(self, code: grpc.StatusCode, details: Optional[str] = None, - initial_metadata: Optional[MetadataType] = None, - trailing_metadata: Optional[MetadataType] = None, + initial_metadata: Optional[Metadata] = None, + trailing_metadata: Optional[Metadata] = None, debug_error_string: Optional[str] = None) -> None: """Constructor. @@ -84,8 +84,8 @@ class AioRpcError(grpc.RpcError): super().__init__(self) self._code = code self._details = details - self._initial_metadata = Metadata(*(initial_metadata or ())) - self._trailing_metadata = Metadata(*(trailing_metadata or ())) + self._initial_metadata = initial_metadata + self._trailing_metadata = trailing_metadata self._debug_error_string = debug_error_string def code(self) -> grpc.StatusCode: @@ -104,7 +104,7 @@ class AioRpcError(grpc.RpcError): """ return self._details - def initial_metadata(self) -> Optional[MetadataType]: + def initial_metadata(self) -> Metadata: """Accesses the initial metadata sent by the server. Returns: @@ -112,7 +112,7 @@ class AioRpcError(grpc.RpcError): """ return self._initial_metadata - def trailing_metadata(self) -> Optional[MetadataType]: + def trailing_metadata(self) -> Metadata: """Accesses the trailing metadata sent by the server. Returns: @@ -141,13 +141,13 @@ class AioRpcError(grpc.RpcError): return self._repr() -def _create_rpc_error(initial_metadata: Optional[MetadataType], +def _create_rpc_error(initial_metadata: Metadata, status: cygrpc.AioRpcStatus) -> AioRpcError: return AioRpcError( _common.CYGRPC_STATUS_CODE_TO_STATUS_CODE[status.code()], status.details(), - initial_metadata, - status.trailing_metadata(), + Metadata.from_tuple(initial_metadata), + Metadata.from_tuple(status.trailing_metadata()), status.debug_error_string(), ) @@ -164,7 +164,7 @@ class Call: _request_serializer: SerializingFunction _response_deserializer: DeserializingFunction - def __init__(self, cython_call: cygrpc._AioCall, metadata: MetadataType, + def __init__(self, cython_call: cygrpc._AioCall, metadata: Metadata, request_serializer: SerializingFunction, response_deserializer: DeserializingFunction, loop: asyncio.AbstractEventLoop) -> None: @@ -204,14 +204,14 @@ class Call: def time_remaining(self) -> Optional[float]: return self._cython_call.time_remaining() - async def initial_metadata(self) -> MetadataType: + async def initial_metadata(self) -> Metadata: raw_metadata_tuple = await self._cython_call.initial_metadata() - return Metadata(*(raw_metadata_tuple or ())) + return Metadata.from_tuple(raw_metadata_tuple) - async def trailing_metadata(self) -> MetadataType: + async def trailing_metadata(self) -> Metadata: raw_metadata_tuple = (await self._cython_call.status()).trailing_metadata() - return Metadata(*(raw_metadata_tuple or ())) + return Metadata.from_tuple(raw_metadata_tuple) async def code(self) -> grpc.StatusCode: cygrpc_code = (await self._cython_call.status()).code() @@ -474,7 +474,7 @@ class UnaryUnaryCall(_UnaryResponseMixin, Call, _base_call.UnaryUnaryCall): # pylint: disable=too-many-arguments def __init__(self, request: RequestType, deadline: Optional[float], - metadata: MetadataType, + metadata: Metadata, credentials: Optional[grpc.CallCredentials], wait_for_ready: Optional[bool], channel: cygrpc.AioChannel, method: bytes, request_serializer: SerializingFunction, @@ -523,7 +523,7 @@ class UnaryStreamCall(_StreamResponseMixin, Call, _base_call.UnaryStreamCall): # pylint: disable=too-many-arguments def __init__(self, request: RequestType, deadline: Optional[float], - metadata: MetadataType, + metadata: Metadata, credentials: Optional[grpc.CallCredentials], wait_for_ready: Optional[bool], channel: cygrpc.AioChannel, method: bytes, request_serializer: SerializingFunction, @@ -563,7 +563,7 @@ class StreamUnaryCall(_StreamRequestMixin, _UnaryResponseMixin, Call, # pylint: disable=too-many-arguments def __init__(self, request_iterator: Optional[RequestIterableType], - deadline: Optional[float], metadata: MetadataType, + deadline: Optional[float], metadata: Metadata, credentials: Optional[grpc.CallCredentials], wait_for_ready: Optional[bool], channel: cygrpc.AioChannel, method: bytes, request_serializer: SerializingFunction, @@ -601,7 +601,7 @@ class StreamStreamCall(_StreamRequestMixin, _StreamResponseMixin, Call, # pylint: disable=too-many-arguments def __init__(self, request_iterator: Optional[RequestIterableType], - deadline: Optional[float], metadata: MetadataType, + deadline: Optional[float], metadata: Metadata, credentials: Optional[grpc.CallCredentials], wait_for_ready: Optional[bool], channel: cygrpc.AioChannel, method: bytes, request_serializer: SerializingFunction, diff --git a/src/python/grpcio/grpc/experimental/aio/_metadata.py b/src/python/grpcio/grpc/experimental/aio/_metadata.py index ff970106748..3230445d58a 100644 --- a/src/python/grpcio/grpc/experimental/aio/_metadata.py +++ b/src/python/grpcio/grpc/experimental/aio/_metadata.py @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. """Implementation of the metadata abstraction for gRPC Asyncio Python.""" -from typing import List, Tuple, Iterator, Any, Text, Union +from typing import List, Tuple, Iterator, Any, Union from collections import abc, OrderedDict -MetadataKey = Text +MetadataKey = str MetadataValue = Union[str, bytes] @@ -37,6 +37,12 @@ class Metadata(abc.Mapping): for md_key, md_value in args: self.add(md_key, md_value) + @classmethod + def from_tuple(cls, raw_metadata: tuple): + if raw_metadata: + return cls(*raw_metadata) + return cls() + def add(self, key: MetadataKey, value: MetadataValue) -> None: self._metadata.setdefault(key, []) self._metadata[key].append(value) diff --git a/src/python/grpcio_tests/tests_aio/unit/_metadata_test.py b/src/python/grpcio_tests/tests_aio/unit/_metadata_test.py index dda58c5ed53..c0594cb06ab 100644 --- a/src/python/grpcio_tests/tests_aio/unit/_metadata_test.py +++ b/src/python/grpcio_tests/tests_aio/unit/_metadata_test.py @@ -119,6 +119,18 @@ class TestTypeMetadata(unittest.TestCase): with self.assertRaises(KeyError): del metadata["other key"] + def test_metadata_from_tuple(self): + scenarios = ( + (None, Metadata()), + (Metadata(), Metadata()), + (self._DEFAULT_DATA, Metadata(*self._DEFAULT_DATA)), + (self._MULTI_ENTRY_DATA, Metadata(*self._MULTI_ENTRY_DATA)), + (Metadata(*self._DEFAULT_DATA), Metadata(*self._DEFAULT_DATA)), + ) + for source, expected in scenarios: + with self.subTest(raw_metadata=source, expected=expected): + self.assertEqual(expected, Metadata.from_tuple(source)) + if __name__ == '__main__': logging.basicConfig() diff --git a/src/python/grpcio_tests/tests_aio/unit/call_test.py b/src/python/grpcio_tests/tests_aio/unit/call_test.py index 94a36a4c070..1961226fa6d 100644 --- a/src/python/grpcio_tests/tests_aio/unit/call_test.py +++ b/src/python/grpcio_tests/tests_aio/unit/call_test.py @@ -102,11 +102,11 @@ class TestUnaryUnaryCall(_MulticallableTestMixin, AioTestBase): async def test_call_initial_metadata_awaitable(self): call = self._stub.UnaryCall(messages_pb2.SimpleRequest()) - self.assertEqual(await call.initial_metadata(), aio.Metadata()) + self.assertEqual(aio.Metadata(), await call.initial_metadata()) async def test_call_trailing_metadata_awaitable(self): call = self._stub.UnaryCall(messages_pb2.SimpleRequest()) - self.assertEqual(await call.trailing_metadata(), aio.Metadata()) + self.assertEqual(aio.Metadata(), await call.trailing_metadata()) async def test_call_initial_metadata_cancelable(self): coro_started = asyncio.Event() @@ -122,7 +122,7 @@ class TestUnaryUnaryCall(_MulticallableTestMixin, AioTestBase): # Test that initial metadata can still be asked thought # a cancellation happened with the previous task - self.assertEqual(await call.initial_metadata(), aio.Metadata()) + self.assertEqual(aio.Metadata(), await call.initial_metadata()) async def test_call_initial_metadata_multiple_waiters(self): call = self._stub.UnaryCall(messages_pb2.SimpleRequest()) @@ -135,7 +135,7 @@ class TestUnaryUnaryCall(_MulticallableTestMixin, AioTestBase): await call expected = [aio.Metadata() for _ in range(2)] - self.assertEqual(await asyncio.gather(*[task1, task2]), expected) + self.assertEqual(expected, await asyncio.gather(*[task1, task2])) async def test_call_code_cancelable(self): coro_started = asyncio.Event() diff --git a/src/python/grpcio_tests/tests_aio/unit/metadata_test.py b/src/python/grpcio_tests/tests_aio/unit/metadata_test.py index 59f5596c70b..822bd134521 100644 --- a/src/python/grpcio_tests/tests_aio/unit/metadata_test.py +++ b/src/python/grpcio_tests/tests_aio/unit/metadata_test.py @@ -57,6 +57,10 @@ _INVALID_METADATA_TEST_CASES = ( TypeError, ((42, 42),), ), + ( + TypeError, + (({}, {}),), + ), ( TypeError, ((None, {}),), From 5a5a5784462c3e3a369f0a75f5ce5e90ce2182bc Mon Sep 17 00:00:00 2001 From: Mariano Anaya Date: Tue, 2 Jun 2020 11:44:14 +0200 Subject: [PATCH 10/80] Fix new metadata tests Using the new aio.Metadata() type instead of tuple. --- .../unit/client_stream_stream_interceptor_test.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/python/grpcio_tests/tests_aio/unit/client_stream_stream_interceptor_test.py b/src/python/grpcio_tests/tests_aio/unit/client_stream_stream_interceptor_test.py index 9ab54b39a6b..ce6a7bc04d6 100644 --- a/src/python/grpcio_tests/tests_aio/unit/client_stream_stream_interceptor_test.py +++ b/src/python/grpcio_tests/tests_aio/unit/client_stream_stream_interceptor_test.py @@ -98,8 +98,8 @@ class TestStreamStreamClientInterceptor(AioTestBase): self.assertEqual(response_cnt, _NUM_STREAM_RESPONSES) self.assertEqual(await call.code(), grpc.StatusCode.OK) - self.assertEqual(await call.initial_metadata(), ()) - self.assertEqual(await call.trailing_metadata(), ()) + self.assertEqual(await call.initial_metadata(), aio.Metadata()) + self.assertEqual(await call.trailing_metadata(), aio.Metadata()) self.assertEqual(await call.details(), '') self.assertEqual(await call.debug_error_string(), '') self.assertEqual(call.cancel(), False) @@ -140,8 +140,8 @@ class TestStreamStreamClientInterceptor(AioTestBase): await call.done_writing() self.assertEqual(await call.code(), grpc.StatusCode.OK) - self.assertEqual(await call.initial_metadata(), ()) - self.assertEqual(await call.trailing_metadata(), ()) + self.assertEqual(await call.initial_metadata(), aio.Metadata()) + self.assertEqual(await call.trailing_metadata(), aio.Metadata()) self.assertEqual(await call.details(), '') self.assertEqual(await call.debug_error_string(), '') self.assertEqual(call.cancel(), False) @@ -183,8 +183,8 @@ class TestStreamStreamClientInterceptor(AioTestBase): await call.done_writing() self.assertEqual(await call.code(), grpc.StatusCode.OK) - self.assertEqual(await call.initial_metadata(), ()) - self.assertEqual(await call.trailing_metadata(), ()) + self.assertEqual(await call.initial_metadata(), aio.Metadata()) + self.assertEqual(await call.trailing_metadata(), aio.Metadata()) self.assertEqual(await call.details(), '') self.assertEqual(await call.debug_error_string(), '') self.assertEqual(call.cancel(), False) From a6bf093af8dcb13a802137d42e965b1b59152587 Mon Sep 17 00:00:00 2001 From: Mariano Anaya Date: Tue, 2 Jun 2020 11:49:57 +0200 Subject: [PATCH 11/80] Use metadata types in the service context Replace the signature to allow methods to use the metadata object. Internally, they'll still wrap the data in a tuple, but the interface makes it clear that the ``aio.Metadata()`` object is supported. Remove the ``tuple()`` conversions done in the tests. --- .../grpc/_cython/_cygrpc/aio/server.pyx.pxi | 16 ++++++++-------- .../grpcio_tests/tests_aio/unit/metadata_test.py | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi index f37769c0038..b842ec6f2ba 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi @@ -143,7 +143,7 @@ cdef class _ServicerContext: self._loop) self._rpc_state.metadata_sent = True - async def send_initial_metadata(self, tuple metadata): + async def send_initial_metadata(self, object metadata): self._rpc_state.raise_for_termination() if self._rpc_state.metadata_sent: @@ -151,7 +151,7 @@ cdef class _ServicerContext: else: await _send_initial_metadata( self._rpc_state, - _augment_metadata(metadata, self._rpc_state.compression_algorithm), + _augment_metadata(tuple(metadata), self._rpc_state.compression_algorithm), _EMPTY_FLAG, self._loop ) @@ -192,8 +192,8 @@ cdef class _ServicerContext: async def abort_with_status(self, object status): await self.abort(status.code, status.details, status.trailing_metadata) - def set_trailing_metadata(self, tuple metadata): - self._rpc_state.trailing_metadata = metadata + def set_trailing_metadata(self, object metadata): + self._rpc_state.trailing_metadata = tuple(metadata) def invocation_metadata(self): return self._rpc_state.invocation_metadata() @@ -233,13 +233,13 @@ cdef class _SyncServicerContext: # Abort should raise an AbortError future.exception() - def send_initial_metadata(self, tuple metadata): + def send_initial_metadata(self, object metadata): future = asyncio.run_coroutine_threadsafe( self._context.send_initial_metadata(metadata), self._loop) future.result() - def set_trailing_metadata(self, tuple metadata): + def set_trailing_metadata(self, object metadata): self._context.set_trailing_metadata(metadata) def invocation_metadata(self): @@ -303,7 +303,7 @@ async def _finish_handler_with_unary_response(RPCState rpc_state, object response_serializer, object loop): """Finishes server method handler with a single response. - + This function executes the application handler, and handles response sending, as well as errors. It is shared between unary-unary and stream-unary handlers. @@ -378,7 +378,7 @@ async def _finish_handler_with_stream_responses(RPCState rpc_state, """ cdef object async_response_generator cdef object response_message - + if inspect.iscoroutinefunction(stream_handler): # Case 1: Coroutine async handler - using reader-writer API # The handler uses reader / writer API, returns None. diff --git a/src/python/grpcio_tests/tests_aio/unit/metadata_test.py b/src/python/grpcio_tests/tests_aio/unit/metadata_test.py index 822bd134521..6fee0c62630 100644 --- a/src/python/grpcio_tests/tests_aio/unit/metadata_test.py +++ b/src/python/grpcio_tests/tests_aio/unit/metadata_test.py @@ -104,13 +104,13 @@ class _TestGenericHandlerForMethods(grpc.GenericRpcHandler): async def _test_server_to_client(request, context): assert _REQUEST == request await context.send_initial_metadata( - tuple(_INITIAL_METADATA_FROM_SERVER_TO_CLIENT)) + _INITIAL_METADATA_FROM_SERVER_TO_CLIENT) return _RESPONSE @staticmethod async def _test_trailing_metadata(request, context): assert _REQUEST == request - context.set_trailing_metadata(tuple(_TRAILING_METADATA)) + context.set_trailing_metadata(_TRAILING_METADATA) return _RESPONSE @staticmethod @@ -119,21 +119,21 @@ class _TestGenericHandlerForMethods(grpc.GenericRpcHandler): assert _common.seen_metadata(_INITIAL_METADATA_FROM_CLIENT_TO_SERVER, context.invocation_metadata()) await context.send_initial_metadata( - tuple(_INITIAL_METADATA_FROM_SERVER_TO_CLIENT)) + _INITIAL_METADATA_FROM_SERVER_TO_CLIENT) yield _RESPONSE - context.set_trailing_metadata(tuple(_TRAILING_METADATA)) + context.set_trailing_metadata(_TRAILING_METADATA) @staticmethod async def _test_stream_unary(request_iterator, context): assert _common.seen_metadata(_INITIAL_METADATA_FROM_CLIENT_TO_SERVER, context.invocation_metadata()) await context.send_initial_metadata( - tuple(_INITIAL_METADATA_FROM_SERVER_TO_CLIENT)) + _INITIAL_METADATA_FROM_SERVER_TO_CLIENT) async for request in request_iterator: assert _REQUEST == request - context.set_trailing_metadata(tuple(_TRAILING_METADATA)) + context.set_trailing_metadata(_TRAILING_METADATA) return _RESPONSE @staticmethod @@ -141,13 +141,13 @@ class _TestGenericHandlerForMethods(grpc.GenericRpcHandler): assert _common.seen_metadata(_INITIAL_METADATA_FROM_CLIENT_TO_SERVER, context.invocation_metadata()) await context.send_initial_metadata( - tuple(_INITIAL_METADATA_FROM_SERVER_TO_CLIENT)) + _INITIAL_METADATA_FROM_SERVER_TO_CLIENT) async for request in request_iterator: assert _REQUEST == request yield _RESPONSE - context.set_trailing_metadata(tuple(_TRAILING_METADATA)) + context.set_trailing_metadata(_TRAILING_METADATA) def service(self, handler_call_details): return self._routing_table.get(handler_call_details.method) From 7b3430ef3ed76f6a5cc15f45e350735009bb1370 Mon Sep 17 00:00:00 2001 From: Mariano Anaya Date: Tue, 2 Jun 2020 11:53:22 +0200 Subject: [PATCH 12/80] Restore test that passes metadata in a list The old interface of accepting the metadata as a list, should be kept due to a backwards incompatibility with a client. The new ``aio.Metadata()`` type supports iteration, so creating a list from it, is possible. --- src/python/grpcio_tests/tests_aio/unit/metadata_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/grpcio_tests/tests_aio/unit/metadata_test.py b/src/python/grpcio_tests/tests_aio/unit/metadata_test.py index 6fee0c62630..9e4de909216 100644 --- a/src/python/grpcio_tests/tests_aio/unit/metadata_test.py +++ b/src/python/grpcio_tests/tests_aio/unit/metadata_test.py @@ -213,7 +213,7 @@ class TestMetadata(AioTestBase): async def test_from_client_to_server_with_list(self): multicallable = self._client.unary_unary(_TEST_CLIENT_TO_SERVER) call = multicallable(_REQUEST, - metadata=_INITIAL_METADATA_FROM_CLIENT_TO_SERVER) + metadata=list(_INITIAL_METADATA_FROM_CLIENT_TO_SERVER)) self.assertEqual(_RESPONSE, await call) self.assertEqual(grpc.StatusCode.OK, await call.code()) From 6e83eb79f48d8abb09fb3cb1c853422e44b52b6b Mon Sep 17 00:00:00 2001 From: Mariano Anaya Date: Tue, 2 Jun 2020 12:38:46 +0200 Subject: [PATCH 13/80] Apply formatting & fix typing --- src/python/grpcio/grpc/experimental/aio/_call.py | 6 +++--- src/python/grpcio_tests/tests_aio/unit/metadata_test.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/python/grpcio/grpc/experimental/aio/_call.py b/src/python/grpcio/grpc/experimental/aio/_call.py index c121bd6b76d..bf5865a3af0 100644 --- a/src/python/grpcio/grpc/experimental/aio/_call.py +++ b/src/python/grpcio/grpc/experimental/aio/_call.py @@ -26,9 +26,9 @@ from grpc._cython import cygrpc from . import _base_call from ._metadata import Metadata -from ._typing import (DeserializingFunction, DoneCallbackType, - MetadatumType, RequestIterableType, RequestType, - ResponseType, SerializingFunction) +from ._typing import (DeserializingFunction, DoneCallbackType, MetadatumType, + RequestIterableType, RequestType, ResponseType, + SerializingFunction) __all__ = 'AioRpcError', 'Call', 'UnaryUnaryCall', 'UnaryStreamCall' diff --git a/src/python/grpcio_tests/tests_aio/unit/metadata_test.py b/src/python/grpcio_tests/tests_aio/unit/metadata_test.py index 9e4de909216..0c8956537ce 100644 --- a/src/python/grpcio_tests/tests_aio/unit/metadata_test.py +++ b/src/python/grpcio_tests/tests_aio/unit/metadata_test.py @@ -212,8 +212,8 @@ class TestMetadata(AioTestBase): async def test_from_client_to_server_with_list(self): multicallable = self._client.unary_unary(_TEST_CLIENT_TO_SERVER) - call = multicallable(_REQUEST, - metadata=list(_INITIAL_METADATA_FROM_CLIENT_TO_SERVER)) + call = multicallable( + _REQUEST, metadata=list(_INITIAL_METADATA_FROM_CLIENT_TO_SERVER)) # pytype: disable=wrong-arg-types self.assertEqual(_RESPONSE, await call) self.assertEqual(grpc.StatusCode.OK, await call.code()) From 84fccf53002e051539251dffc6738ef9a470062b Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Thu, 4 Jun 2020 11:06:49 -0700 Subject: [PATCH 14/80] Update default xds test qps to 100 --- doc/xds-test-descriptions.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/xds-test-descriptions.md b/doc/xds-test-descriptions.md index 46414384e95..3e2798d1425 100644 --- a/doc/xds-test-descriptions.md +++ b/doc/xds-test-descriptions.md @@ -90,7 +90,7 @@ This test verifies that every backend receives traffic. Client parameters: 1. --num_channels=1 -1. --qps=10 +1. --qps=100 1. --fail_on_failed_rpc=true Load balancer configuration: @@ -109,7 +109,7 @@ robin policy. Client parameters: 1. --num_channels=1 -1. --qps=10 +1. --qps=100 1. --fail_on_failed_rpc=true Load balancer configuration: @@ -129,7 +129,7 @@ of backends that is stopped and then resumed. Client parameters: 1. --num_channels=1 -1. --qps=10 +1. --qps=100 Load balancer configuration: @@ -161,7 +161,7 @@ all backends in the primary locality fail. Client parameters: 1. --num_channels=1 -1. --qps=10 +1. --qps=100 Load balancer configuration: @@ -197,7 +197,7 @@ changes to this test case. Client parameters: 1. --num_channels=1 -1. --qps=10 +1. --qps=100 Load balancer configuration: @@ -224,7 +224,7 @@ same zone receive traffic. Client parameters: 1. --num_channels=1 -1. --qps=10 +1. --qps=100 1. --fail_on_failed_rpc=true Load balancer configuration: @@ -249,7 +249,7 @@ after removal of another instance group in the same zone. Client parameters: 1. --num_channels=1 -1. --qps=10 +1. --qps=100 Load balancer configuration: @@ -273,7 +273,7 @@ to the new backends. Client parameters: 1. --num_channels=1 -1. --qps=10 +1. --qps=100 1. --fail_on_failed_rpc=true Load balancer configuration: From a1a06a555d62352083e2df0539418180d3657871 Mon Sep 17 00:00:00 2001 From: Menghan Li Date: Thu, 4 Jun 2020 11:47:42 -0700 Subject: [PATCH 15/80] xds interop descriptions: add test traffic_splitting to spec --- doc/xds-test-descriptions.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/doc/xds-test-descriptions.md b/doc/xds-test-descriptions.md index 46414384e95..a106f1354ec 100644 --- a/doc/xds-test-descriptions.md +++ b/doc/xds-test-descriptions.md @@ -291,3 +291,31 @@ Test driver asserts: 1. All RPCs are directed to the new backend service. +### traffic_splitting + +This test verifies that the traffic will be distributed between backend +services with the correct weights when route action is set to weighted +backend services. + +Client parameters: + +1. --num_channels=1 +1. --qps=100 + +Load balancer configuration: + +1. One MIG with one backend + +Assert: + +1. Once all backends receive at least one RPC, the following 1000 RPCs are +all sent to MIG_a. + +The test driver adds a new MIG with 1 backend, and changes the route action +to weighted backend services with {a: 20, b: 80}. + +Assert: + +1. Once all backends receive at least one RPC, the following 1000 RPCs are +distributed across the 2 backends as a: 20, b: 80. + From 36f79adaf9cda4179a4612a07ecf92a95b3616e8 Mon Sep 17 00:00:00 2001 From: Mariano Anaya Date: Mon, 8 Jun 2020 16:52:31 +0200 Subject: [PATCH 16/80] Remove references to the old MetadataType --- .../grpc/experimental/aio/_base_call.py | 7 ++-- .../grpc/experimental/aio/_base_channel.py | 13 ++++---- .../grpc/experimental/aio/_base_server.py | 11 ++++--- .../grpcio/grpc/experimental/aio/_channel.py | 10 +++--- .../grpc/experimental/aio/_interceptor.py | 33 ++++++++++--------- .../grpcio_tests/tests_aio/unit/_common.py | 7 ++-- 6 files changed, 42 insertions(+), 39 deletions(-) diff --git a/src/python/grpcio/grpc/experimental/aio/_base_call.py b/src/python/grpcio/grpc/experimental/aio/_base_call.py index 214e208c005..c07b4ca6b14 100644 --- a/src/python/grpcio/grpc/experimental/aio/_base_call.py +++ b/src/python/grpcio/grpc/experimental/aio/_base_call.py @@ -23,8 +23,9 @@ from typing import AsyncIterable, Awaitable, Generic, Optional, Union import grpc -from ._typing import (DoneCallbackType, EOFType, MetadataType, RequestType, +from ._typing import (DoneCallbackType, EOFType, RequestType, ResponseType) +from ._metadata import Metadata __all__ = 'RpcContext', 'Call', 'UnaryUnaryCall', 'UnaryStreamCall' @@ -86,7 +87,7 @@ class Call(RpcContext, metaclass=ABCMeta): """The abstract base class of an RPC on the client-side.""" @abstractmethod - async def initial_metadata(self) -> MetadataType: + async def initial_metadata(self) -> Metadata: """Accesses the initial metadata sent by the server. Returns: @@ -94,7 +95,7 @@ class Call(RpcContext, metaclass=ABCMeta): """ @abstractmethod - async def trailing_metadata(self) -> MetadataType: + async def trailing_metadata(self) -> Metadata: """Accesses the trailing metadata sent by the server. Returns: diff --git a/src/python/grpcio/grpc/experimental/aio/_base_channel.py b/src/python/grpcio/grpc/experimental/aio/_base_channel.py index 33efa7789cb..4b4ea1355b4 100644 --- a/src/python/grpcio/grpc/experimental/aio/_base_channel.py +++ b/src/python/grpcio/grpc/experimental/aio/_base_channel.py @@ -19,10 +19,9 @@ from typing import Any, Optional import grpc from . import _base_call -from ._typing import (DeserializingFunction, MetadataType, RequestIterableType, +from ._typing import (DeserializingFunction, RequestIterableType, SerializingFunction) - -_IMMUTABLE_EMPTY_TUPLE = tuple() +from ._metadata import Metadata class UnaryUnaryMultiCallable(abc.ABC): @@ -33,7 +32,7 @@ class UnaryUnaryMultiCallable(abc.ABC): request: Any, *, timeout: Optional[float] = None, - metadata: Optional[MetadataType] = _IMMUTABLE_EMPTY_TUPLE, + metadata: Optional[Metadata] = None, credentials: Optional[grpc.CallCredentials] = None, wait_for_ready: Optional[bool] = None, compression: Optional[grpc.Compression] = None @@ -71,7 +70,7 @@ class UnaryStreamMultiCallable(abc.ABC): request: Any, *, timeout: Optional[float] = None, - metadata: Optional[MetadataType] = _IMMUTABLE_EMPTY_TUPLE, + metadata: Optional[Metadata] = None, credentials: Optional[grpc.CallCredentials] = None, wait_for_ready: Optional[bool] = None, compression: Optional[grpc.Compression] = None @@ -108,7 +107,7 @@ class StreamUnaryMultiCallable(abc.ABC): def __call__(self, request_iterator: Optional[RequestIterableType] = None, timeout: Optional[float] = None, - metadata: Optional[MetadataType] = _IMMUTABLE_EMPTY_TUPLE, + metadata: Optional[Metadata] = None, credentials: Optional[grpc.CallCredentials] = None, wait_for_ready: Optional[bool] = None, compression: Optional[grpc.Compression] = None @@ -146,7 +145,7 @@ class StreamStreamMultiCallable(abc.ABC): def __call__(self, request_iterator: Optional[RequestIterableType] = None, timeout: Optional[float] = None, - metadata: Optional[MetadataType] = _IMMUTABLE_EMPTY_TUPLE, + metadata: Optional[Metadata] = None, credentials: Optional[grpc.CallCredentials] = None, wait_for_ready: Optional[bool] = None, compression: Optional[grpc.Compression] = None diff --git a/src/python/grpcio/grpc/experimental/aio/_base_server.py b/src/python/grpcio/grpc/experimental/aio/_base_server.py index 72e4288c94f..842e9b15c9e 100644 --- a/src/python/grpcio/grpc/experimental/aio/_base_server.py +++ b/src/python/grpcio/grpc/experimental/aio/_base_server.py @@ -18,7 +18,8 @@ from typing import Generic, Optional, Sequence import grpc -from ._typing import MetadataType, RequestType, ResponseType +from ._typing import RequestType, ResponseType +from ._metadata import Metadata class Server(abc.ABC): @@ -158,7 +159,7 @@ class ServicerContext(Generic[RequestType, ResponseType], abc.ABC): @abc.abstractmethod async def send_initial_metadata(self, - initial_metadata: MetadataType) -> None: + initial_metadata: Metadata) -> None: """Sends the initial metadata value to the client. This method need not be called by implementations if they have no @@ -170,7 +171,7 @@ class ServicerContext(Generic[RequestType, ResponseType], abc.ABC): @abc.abstractmethod async def abort(self, code: grpc.StatusCode, details: str, - trailing_metadata: MetadataType) -> None: + trailing_metadata: Metadata) -> None: """Raises an exception to terminate the RPC with a non-OK status. The code and details passed as arguments will supercede any existing @@ -191,7 +192,7 @@ class ServicerContext(Generic[RequestType, ResponseType], abc.ABC): @abc.abstractmethod async def set_trailing_metadata(self, - trailing_metadata: MetadataType) -> None: + trailing_metadata: Metadata) -> None: """Sends the trailing metadata for the RPC. This method need not be called by implementations if they have no @@ -202,7 +203,7 @@ class ServicerContext(Generic[RequestType, ResponseType], abc.ABC): """ @abc.abstractmethod - def invocation_metadata(self) -> Optional[MetadataType]: + def invocation_metadata(self) -> Optional[Metadata]: """Accesses the metadata from the sent by the client. Returns: diff --git a/src/python/grpcio/grpc/experimental/aio/_channel.py b/src/python/grpcio/grpc/experimental/aio/_channel.py index 3ac12bf6139..1995db13bf5 100644 --- a/src/python/grpcio/grpc/experimental/aio/_channel.py +++ b/src/python/grpcio/grpc/experimental/aio/_channel.py @@ -30,7 +30,7 @@ from ._interceptor import ( UnaryUnaryClientInterceptor, UnaryStreamClientInterceptor, StreamUnaryClientInterceptor, StreamStreamClientInterceptor) from ._metadata import Metadata -from ._typing import (ChannelArgumentType, DeserializingFunction, MetadataType, +from ._typing import (ChannelArgumentType, DeserializingFunction, SerializingFunction, RequestIterableType) from ._utils import _timeout_to_deadline @@ -109,7 +109,7 @@ class UnaryUnaryMultiCallable(_BaseMultiCallable, request: Any, *, timeout: Optional[float] = None, - metadata: Optional[MetadataType] = None, + metadata: Optional[Metadata] = None, credentials: Optional[grpc.CallCredentials] = None, wait_for_ready: Optional[bool] = None, compression: Optional[grpc.Compression] = None @@ -139,7 +139,7 @@ class UnaryStreamMultiCallable(_BaseMultiCallable, request: Any, *, timeout: Optional[float] = None, - metadata: Optional[MetadataType] = None, + metadata: Optional[Metadata] = None, credentials: Optional[grpc.CallCredentials] = None, wait_for_ready: Optional[bool] = None, compression: Optional[grpc.Compression] = None @@ -169,7 +169,7 @@ class StreamUnaryMultiCallable(_BaseMultiCallable, def __call__(self, request_iterator: Optional[RequestIterableType] = None, timeout: Optional[float] = None, - metadata: Optional[MetadataType] = None, + metadata: Optional[Metadata] = None, credentials: Optional[grpc.CallCredentials] = None, wait_for_ready: Optional[bool] = None, compression: Optional[grpc.Compression] = None @@ -199,7 +199,7 @@ class StreamStreamMultiCallable(_BaseMultiCallable, def __call__(self, request_iterator: Optional[RequestIterableType] = None, timeout: Optional[float] = None, - metadata: Optional[MetadataType] = None, + metadata: Optional[Metadata] = None, credentials: Optional[grpc.CallCredentials] = None, wait_for_ready: Optional[bool] = None, compression: Optional[grpc.Compression] = None diff --git a/src/python/grpcio/grpc/experimental/aio/_interceptor.py b/src/python/grpcio/grpc/experimental/aio/_interceptor.py index 8a28a61c8ba..c8f185afb56 100644 --- a/src/python/grpcio/grpc/experimental/aio/_interceptor.py +++ b/src/python/grpcio/grpc/experimental/aio/_interceptor.py @@ -27,8 +27,9 @@ from ._call import _RPC_ALREADY_FINISHED_DETAILS, _RPC_HALF_CLOSED_DETAILS from ._call import _API_STYLE_ERROR from ._utils import _timeout_to_deadline from ._typing import (RequestType, SerializingFunction, DeserializingFunction, - MetadataType, ResponseType, DoneCallbackType, + ResponseType, DoneCallbackType, RequestIterableType, ResponseIterableType) +from ._metadata import Metadata _LOCAL_CANCELLATION_DETAILS = 'Locally cancelled by application!' @@ -82,7 +83,7 @@ class ClientCallDetails( method: str timeout: Optional[float] - metadata: Optional[MetadataType] + metadata: Optional[Metadata] credentials: Optional[grpc.CallCredentials] wait_for_ready: Optional[bool] @@ -370,7 +371,7 @@ class InterceptedCall: def time_remaining(self) -> Optional[float]: raise NotImplementedError() - async def initial_metadata(self) -> Optional[MetadataType]: + async def initial_metadata(self) -> Optional[Metadata]: try: call = await self._interceptors_task except AioRpcError as err: @@ -380,7 +381,7 @@ class InterceptedCall: return await call.initial_metadata() - async def trailing_metadata(self) -> Optional[MetadataType]: + async def trailing_metadata(self) -> Optional[Metadata]: try: call = await self._interceptors_task except AioRpcError as err: @@ -556,7 +557,7 @@ class InterceptedUnaryUnaryCall(_InterceptedUnaryResponseMixin, InterceptedCall, # pylint: disable=too-many-arguments def __init__(self, interceptors: Sequence[UnaryUnaryClientInterceptor], request: RequestType, timeout: Optional[float], - metadata: MetadataType, + metadata: Metadata, credentials: Optional[grpc.CallCredentials], wait_for_ready: Optional[bool], channel: cygrpc.AioChannel, method: bytes, request_serializer: SerializingFunction, @@ -573,7 +574,7 @@ class InterceptedUnaryUnaryCall(_InterceptedUnaryResponseMixin, InterceptedCall, # pylint: disable=too-many-arguments async def _invoke(self, interceptors: Sequence[UnaryUnaryClientInterceptor], method: bytes, timeout: Optional[float], - metadata: Optional[MetadataType], + metadata: Optional[Metadata], credentials: Optional[grpc.CallCredentials], wait_for_ready: Optional[bool], request: RequestType, request_serializer: SerializingFunction, @@ -628,7 +629,7 @@ class InterceptedUnaryStreamCall(_InterceptedStreamResponseMixin, # pylint: disable=too-many-arguments def __init__(self, interceptors: Sequence[UnaryStreamClientInterceptor], request: RequestType, timeout: Optional[float], - metadata: MetadataType, + metadata: Metadata, credentials: Optional[grpc.CallCredentials], wait_for_ready: Optional[bool], channel: cygrpc.AioChannel, method: bytes, request_serializer: SerializingFunction, @@ -647,7 +648,7 @@ class InterceptedUnaryStreamCall(_InterceptedStreamResponseMixin, # pylint: disable=too-many-arguments async def _invoke(self, interceptors: Sequence[UnaryUnaryClientInterceptor], method: bytes, timeout: Optional[float], - metadata: Optional[MetadataType], + metadata: Optional[Metadata], credentials: Optional[grpc.CallCredentials], wait_for_ready: Optional[bool], request: RequestType, request_serializer: SerializingFunction, @@ -712,7 +713,7 @@ class InterceptedStreamUnaryCall(_InterceptedUnaryResponseMixin, # pylint: disable=too-many-arguments def __init__(self, interceptors: Sequence[StreamUnaryClientInterceptor], request_iterator: Optional[RequestIterableType], - timeout: Optional[float], metadata: MetadataType, + timeout: Optional[float], metadata: Metadata, credentials: Optional[grpc.CallCredentials], wait_for_ready: Optional[bool], channel: cygrpc.AioChannel, method: bytes, request_serializer: SerializingFunction, @@ -731,7 +732,7 @@ class InterceptedStreamUnaryCall(_InterceptedUnaryResponseMixin, async def _invoke( self, interceptors: Sequence[StreamUnaryClientInterceptor], method: bytes, timeout: Optional[float], - metadata: Optional[MetadataType], + metadata: Optional[Metadata], credentials: Optional[grpc.CallCredentials], wait_for_ready: Optional[bool], request_iterator: RequestIterableType, @@ -783,7 +784,7 @@ class InterceptedStreamStreamCall(_InterceptedStreamResponseMixin, # pylint: disable=too-many-arguments def __init__(self, interceptors: Sequence[StreamStreamClientInterceptor], request_iterator: Optional[RequestIterableType], - timeout: Optional[float], metadata: MetadataType, + timeout: Optional[float], metadata: Metadata, credentials: Optional[grpc.CallCredentials], wait_for_ready: Optional[bool], channel: cygrpc.AioChannel, method: bytes, request_serializer: SerializingFunction, @@ -804,7 +805,7 @@ class InterceptedStreamStreamCall(_InterceptedStreamResponseMixin, async def _invoke( self, interceptors: Sequence[StreamStreamClientInterceptor], method: bytes, timeout: Optional[float], - metadata: Optional[MetadataType], + metadata: Optional[Metadata], credentials: Optional[grpc.CallCredentials], wait_for_ready: Optional[bool], request_iterator: RequestIterableType, @@ -876,10 +877,10 @@ class UnaryUnaryCallResponse(_base_call.UnaryUnaryCall): def time_remaining(self) -> Optional[float]: raise NotImplementedError() - async def initial_metadata(self) -> Optional[MetadataType]: + async def initial_metadata(self) -> Optional[Metadata]: return None - async def trailing_metadata(self) -> Optional[MetadataType]: + async def trailing_metadata(self) -> Optional[Metadata]: return None async def code(self) -> grpc.StatusCode: @@ -928,10 +929,10 @@ class _StreamCallResponseIterator: def time_remaining(self) -> Optional[float]: return self._call.time_remaining() - async def initial_metadata(self) -> Optional[MetadataType]: + async def initial_metadata(self) -> Optional[Metadata]: return await self._call.initial_metadata() - async def trailing_metadata(self) -> Optional[MetadataType]: + async def trailing_metadata(self) -> Optional[Metadata]: return await self._call.trailing_metadata() async def code(self) -> grpc.StatusCode: diff --git a/src/python/grpcio_tests/tests_aio/unit/_common.py b/src/python/grpcio_tests/tests_aio/unit/_common.py index a4a9236069c..7fdd120e31b 100644 --- a/src/python/grpcio_tests/tests_aio/unit/_common.py +++ b/src/python/grpcio_tests/tests_aio/unit/_common.py @@ -16,17 +16,18 @@ import asyncio import grpc from typing import AsyncIterable from grpc.experimental import aio -from grpc.experimental.aio._typing import MetadataType, MetadatumType, MetadataKey, MetadataValue +from grpc.experimental.aio._typing import MetadatumType, MetadataKey, MetadataValue +from grpc.experimental.aio._metadata import Metadata from tests.unit.framework.common import test_constants -def seen_metadata(expected: MetadataType, actual: MetadataType): +def seen_metadata(expected: Metadata, actual: Metadata): return not bool(set(tuple(expected)) - set(tuple(actual))) def seen_metadatum(expected_key: MetadataKey, expected_value: MetadataValue, - actual: MetadataType) -> bool: + actual: Metadata) -> bool: obtained = actual[expected_key] return obtained == expected_value From 18e0f9f53313a316dd44a0bb94ef3dff8922f1ee Mon Sep 17 00:00:00 2001 From: Mariano Anaya Date: Mon, 8 Jun 2020 17:02:32 +0200 Subject: [PATCH 17/80] Remove metadata as optional from AioRpcError --- src/python/grpcio/grpc/experimental/aio/_call.py | 8 ++++---- .../grpcio_tests/tests_aio/unit/aio_rpc_error_test.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/python/grpcio/grpc/experimental/aio/_call.py b/src/python/grpcio/grpc/experimental/aio/_call.py index bf5865a3af0..ba229f35c39 100644 --- a/src/python/grpcio/grpc/experimental/aio/_call.py +++ b/src/python/grpcio/grpc/experimental/aio/_call.py @@ -67,9 +67,9 @@ class AioRpcError(grpc.RpcError): def __init__(self, code: grpc.StatusCode, + initial_metadata: Metadata, + trailing_metadata: Metadata, details: Optional[str] = None, - initial_metadata: Optional[Metadata] = None, - trailing_metadata: Optional[Metadata] = None, debug_error_string: Optional[str] = None) -> None: """Constructor. @@ -145,10 +145,10 @@ def _create_rpc_error(initial_metadata: Metadata, status: cygrpc.AioRpcStatus) -> AioRpcError: return AioRpcError( _common.CYGRPC_STATUS_CODE_TO_STATUS_CODE[status.code()], - status.details(), Metadata.from_tuple(initial_metadata), Metadata.from_tuple(status.trailing_metadata()), - status.debug_error_string(), + details=status.details(), + debug_error_string=status.debug_error_string(), ) diff --git a/src/python/grpcio_tests/tests_aio/unit/aio_rpc_error_test.py b/src/python/grpcio_tests/tests_aio/unit/aio_rpc_error_test.py index 9bd652a43a6..416c51a7080 100644 --- a/src/python/grpcio_tests/tests_aio/unit/aio_rpc_error_test.py +++ b/src/python/grpcio_tests/tests_aio/unit/aio_rpc_error_test.py @@ -33,9 +33,9 @@ class TestAioRpcError(unittest.TestCase): def test_attributes(self): aio_rpc_error = AioRpcError(grpc.StatusCode.CANCELLED, - 'details', initial_metadata=_TEST_INITIAL_METADATA, trailing_metadata=_TEST_TRAILING_METADATA, + details="details", debug_error_string=_TEST_DEBUG_ERROR_STRING) self.assertEqual(aio_rpc_error.code(), grpc.StatusCode.CANCELLED) self.assertEqual(aio_rpc_error.details(), 'details') From 46c13cd8701ebd0c275ec994e1e902107d88bae5 Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Mon, 8 Jun 2020 18:40:11 -0700 Subject: [PATCH 18/80] Add a flag to interop soak tests to exit early and fail if X seconds have passed and we're not done yet. --- test/cpp/interop/client.cc | 12 ++++++-- test/cpp/interop/interop_client.cc | 44 +++++++++++++++++++++++++----- test/cpp/interop/interop_client.h | 9 ++++-- 3 files changed, 52 insertions(+), 13 deletions(-) diff --git a/test/cpp/interop/client.cc b/test/cpp/interop/client.cc index a23c5ddb6d5..a6c406a77ed 100644 --- a/test/cpp/interop/client.cc +++ b/test/cpp/interop/client.cc @@ -96,7 +96,11 @@ DEFINE_int32(soak_max_failures, 0, "per-iteration max acceptable latency)."); DEFINE_int32(soak_per_iteration_max_acceptable_latency_ms, 0, "The number of milliseconds a single iteration in the two soak " - "tests (rpc_soak and channel_soak) is allowed to take."); + "tests (rpc_soak and channel_soak) should take."); +DEFINE_int32(soak_overall_deadline_seconds, 0, + "The overall number of seconds after which a soak test should " + "stop and fail, if the desired number of iterations have not yet " + "completed."); DEFINE_int32(iteration_interval, 10, "The interval in seconds between rpcs. This is used by " "long_connection test"); @@ -265,11 +269,13 @@ int main(int argc, char** argv) { actions["channel_soak"] = std::bind(&grpc::testing::InteropClient::DoChannelSoakTest, &client, FLAGS_soak_iterations, FLAGS_soak_max_failures, - FLAGS_soak_per_iteration_max_acceptable_latency_ms); + FLAGS_soak_per_iteration_max_acceptable_latency_ms, + FLAGS_soak_overall_deadline_seconds); actions["rpc_soak"] = std::bind(&grpc::testing::InteropClient::DoRpcSoakTest, &client, FLAGS_soak_iterations, FLAGS_soak_max_failures, - FLAGS_soak_per_iteration_max_acceptable_latency_ms); + FLAGS_soak_per_iteration_max_acceptable_latency_ms, + FLAGS_soak_overall_deadline_seconds); actions["long_lived_channel"] = std::bind(&grpc::testing::InteropClient::DoLongLivedChannelTest, &client, FLAGS_soak_iterations, FLAGS_iteration_interval); diff --git a/test/cpp/interop/interop_client.cc b/test/cpp/interop/interop_client.cc index e889d2424c5..7c7f9fdd458 100644 --- a/test/cpp/interop/interop_client.cc +++ b/test/cpp/interop/interop_client.cc @@ -1106,16 +1106,25 @@ InteropClient::PerformOneSoakTestIteration( void InteropClient::PerformSoakTest( const bool reset_channel_per_iteration, const int32_t soak_iterations, const int32_t max_failures, - const int32_t max_acceptable_per_iteration_latency_ms) { + const int32_t max_acceptable_per_iteration_latency_ms, + const int32_t overall_deadline_seconds) { std::vector> results; grpc_histogram* latencies_ms_histogram = grpc_histogram_create( 1 /* resolution */, 500 * 1e3 /* largest bucket; 500 seconds is unlikely */); - for (int i = 0; i < soak_iterations; ++i) { + gpr_timespec overall_deadline = gpr_time_add( + gpr_now(GPR_CLOCK_MONOTONIC), + gpr_time_from_seconds(overall_deadline_seconds, GPR_TIMESPAN)); + int32_t iterations_ran = 0; + for (int i = 0; + i < soak_iterations && + gpr_time_cmp(gpr_now(GPR_CLOCK_MONOTONIC), overall_deadline) < 0; + ++i) { auto result = PerformOneSoakTestIteration( reset_channel_per_iteration, max_acceptable_per_iteration_latency_ms); results.push_back(result); grpc_histogram_add(latencies_ms_histogram, std::get<1>(result)); + iterations_ran++; } int total_failures = 0; for (size_t i = 0; i < results.size(); i++) { @@ -1137,7 +1146,24 @@ void InteropClient::PerformSoakTest( grpc_histogram_percentile(latencies_ms_histogram, 90); double latency_ms_worst = grpc_histogram_maximum(latencies_ms_histogram); grpc_histogram_destroy(latencies_ms_histogram); - if (total_failures > max_failures) { + if (iterations_ran < soak_iterations) { + gpr_log( + GPR_ERROR, + "soak test consumed all %d seconds of time and quit early, only " + "having ran %d out of desired %d iterations. " + "total_failures: %d. " + "max_failures_threshold: %d. " + "median_soak_iteration_latency: %lf ms. " + "90th_soak_iteration_latency: %lf ms. " + "worst_soak_iteration_latency: %lf ms. " + "Some or all of the iterations that did run were unexpectedly slow. " + "See breakdown above for which iterations succeeded, failed, and " + "why for more info.", + overall_deadline_seconds, iterations_ran, soak_iterations, + total_failures, max_failures, latency_ms_median, latency_ms_90th, + latency_ms_worst); + GPR_ASSERT(0); + } else if (total_failures > max_failures) { gpr_log(GPR_ERROR, "soak test ran: %d iterations. total_failures: %d exceeds " "max_failures_threshold: %d. " @@ -1165,23 +1191,27 @@ void InteropClient::PerformSoakTest( bool InteropClient::DoRpcSoakTest( int32_t soak_iterations, int32_t max_failures, - int64_t max_acceptable_per_iteration_latency_ms) { + int64_t max_acceptable_per_iteration_latency_ms, + int32_t overall_deadline_seconds) { gpr_log(GPR_DEBUG, "Sending %d RPCs...", soak_iterations); GPR_ASSERT(soak_iterations > 0); PerformSoakTest(false /* reset channel per iteration */, soak_iterations, - max_failures, max_acceptable_per_iteration_latency_ms); + max_failures, max_acceptable_per_iteration_latency_ms, + overall_deadline_seconds); gpr_log(GPR_DEBUG, "rpc_soak test done."); return true; } bool InteropClient::DoChannelSoakTest( int32_t soak_iterations, int32_t max_failures, - int64_t max_acceptable_per_iteration_latency_ms) { + int64_t max_acceptable_per_iteration_latency_ms, + int32_t overall_deadline_seconds) { gpr_log(GPR_DEBUG, "Sending %d RPCs, tearing down the channel each time...", soak_iterations); GPR_ASSERT(soak_iterations > 0); PerformSoakTest(true /* reset channel per iteration */, soak_iterations, - max_failures, max_acceptable_per_iteration_latency_ms); + max_failures, max_acceptable_per_iteration_latency_ms, + overall_deadline_seconds); gpr_log(GPR_DEBUG, "channel_soak test done."); return true; } diff --git a/test/cpp/interop/interop_client.h b/test/cpp/interop/interop_client.h index 2dc0d8047ec..3d334f748fe 100644 --- a/test/cpp/interop/interop_client.h +++ b/test/cpp/interop/interop_client.h @@ -77,9 +77,11 @@ class InteropClient { // but at some point in the future, might be codified and implemented in all // languages bool DoChannelSoakTest(int32_t soak_iterations, int32_t max_failures, - int64_t max_acceptable_per_iteration_latency_ms); + int64_t max_acceptable_per_iteration_latency_ms, + int32_t overall_deadline_seconds); bool DoRpcSoakTest(int32_t soak_iterations, int32_t max_failures, - int64_t max_acceptable_per_iteration_latency_ms); + int64_t max_acceptable_per_iteration_latency_ms, + int32_t overall_deadline_seconds); bool DoLongLivedChannelTest(int32_t soak_iterations, int32_t iteration_interval); @@ -137,7 +139,8 @@ class InteropClient { void PerformSoakTest(const bool reset_channel_per_iteration, const int32_t soak_iterations, const int32_t max_failures, - const int32_t max_acceptable_per_iteration_latency_ms); + const int32_t max_acceptable_per_iteration_latency_ms, + const int32_t overall_deadline_seconds); ServiceStub serviceStub_; /// If true, abort() is not called for transient failures From f5f44d86a3cf1bf6200e7817360879afce3a4cf4 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 9 Jun 2020 15:31:27 -0700 Subject: [PATCH 19/80] no unref when there's no aborted allocations --- src/core/lib/iomgr/resource_quota.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/resource_quota.cc b/src/core/lib/iomgr/resource_quota.cc index b5df53ecdb8..967d0b7e4f4 100644 --- a/src/core/lib/iomgr/resource_quota.cc +++ b/src/core/lib/iomgr/resource_quota.cc @@ -334,7 +334,9 @@ static bool rq_alloc(grpc_resource_quota* resource_quota) { resource_user->free_pool += aborted_allocations; grpc_core::ExecCtx::RunList(DEBUG_LOCATION, &resource_user->on_allocated); gpr_mu_unlock(&resource_user->mu); - ru_unref_by(resource_user, static_cast(aborted_allocations)); + if (aborted_allocations > 0) { + ru_unref_by(resource_user, static_cast(aborted_allocations)); + } continue; } if (resource_user->free_pool < 0 && From b19f287144d86944613457beada43ebb1b70048c Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 10 Jun 2020 07:03:53 -0400 Subject: [PATCH 20/80] use Microsoft.NETFramework.ReferenceAssemblies for Grpc.Tools --- .../Grpc.Tools.Tests/Grpc.Tools.Tests.csproj | 15 +++++++-------- src/csharp/Grpc.Tools/Grpc.Tools.csproj | 15 +++++++-------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj b/src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj index bc32e375860..ae529b1bbbd 100644 --- a/src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj +++ b/src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj @@ -7,14 +7,13 @@ - - - /usr/lib/mono/4.5-api - /usr/local/lib/mono/4.5-api - /Library/Frameworks/Mono.framework/Versions/Current/lib/mono/4.5-api - + + + + all + runtime; build; native; contentfiles; analyzers + + diff --git a/src/csharp/Grpc.Tools/Grpc.Tools.csproj b/src/csharp/Grpc.Tools/Grpc.Tools.csproj index e73f7b7deae..e4d534768c2 100644 --- a/src/csharp/Grpc.Tools/Grpc.Tools.csproj +++ b/src/csharp/Grpc.Tools/Grpc.Tools.csproj @@ -7,14 +7,13 @@ net45;netstandard1.3 - - - /usr/lib/mono/4.5-api - /usr/local/lib/mono/4.5-api - /Library/Frameworks/Mono.framework/Versions/Current/lib/mono/4.5-api - + + + + all + runtime; build; native; contentfiles; analyzers + + From 47ae4cd5d9eda6fcc3fb57658ae75135a9a6137a Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 10 Jun 2020 11:49:42 -0400 Subject: [PATCH 21/80] move Mono msbuild test hack to C# code --- .../Grpc.Tools.Tests/Grpc.Tools.Tests.csproj | 32 ---------- .../Grpc.Tools.Tests/MsBuildAssemblyHelper.cs | 63 +++++++++++++++++++ src/csharp/Grpc.Tools.Tests/NUnitMain.cs | 9 ++- 3 files changed, 69 insertions(+), 35 deletions(-) create mode 100644 src/csharp/Grpc.Tools.Tests/MsBuildAssemblyHelper.cs diff --git a/src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj b/src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj index ae529b1bbbd..b25bc8d953e 100644 --- a/src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj +++ b/src/csharp/Grpc.Tools.Tests/Grpc.Tools.Tests.csproj @@ -37,36 +37,4 @@ - - - - - - - - - {HintPathFromItem};{TargetFrameworkDirectory};{RawFileName} - - <_MSBuildAssemblyPath Condition=" '$(MSBuildRuntimeType)' != 'Core' " - >$(MSBuildToolsPath) - - <_MSBuildAssemblyPath Condition=" '$(MSBuildRuntimeType)' == 'Core' " - >$(FrameworkPathOverride)/../msbuild/$(MSBuildToolsVersion)/bin - - diff --git a/src/csharp/Grpc.Tools.Tests/MsBuildAssemblyHelper.cs b/src/csharp/Grpc.Tools.Tests/MsBuildAssemblyHelper.cs new file mode 100644 index 00000000000..c24cddb2348 --- /dev/null +++ b/src/csharp/Grpc.Tools.Tests/MsBuildAssemblyHelper.cs @@ -0,0 +1,63 @@ +#region Copyright notice and license + +// Copyright 2018 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#endregion + +using System.Reflection; +using NUnitLite; +using System; +using System.IO; +using System.Runtime.InteropServices; + +namespace Grpc.Tools.Tests +{ + static class MsBuildAssemblyHelper + { + [DllImport("__Internal")] + extern static void mono_set_assemblies_path(string path); + + public static void TweakAssemblyPathIfOnMono() + { + // Below is a hack to allow the tests to run under Mono Framework build. + // Mono unfortunately comes with broken Microsoft.Build.* assemblies installed in + // the GAC, so we need to tweak the assembly search path to make sure the right + // msbuild assemblies are loaded (and the tests work). +#if NET45 + // only run this under .NET framework; under mono + bool isMono = Type.GetType("Mono.Runtime") != null; + if (isMono) + { + var mscorlibDir = Path.GetDirectoryName(typeof(Array).Assembly.Location); + // Construct the location of MsBuild assemblies from the location of mscorlib assembly. + var msbuildToolPath = Path.Combine(mscorlibDir, "..", "msbuild", "Current", "bin"); + + // To make sure we've constructed the right path, make sure the assemblies we're interested + // in are there. + foreach(var assemblyName in new [] {"Microsoft.Build.Framework.dll", "Microsoft.Build.Utilities.v4.0.dll", "Microsoft.Build.Utilities.Core.dll"}) + { + if (!File.Exists(Path.Combine(msbuildToolPath, assemblyName))) + { + throw new InvalidOperationException($"Could not locate assembly {assemblyName} under {msbuildToolPath}"); + } + } + // Normally the assembly search path can be changed by MONO_PATH environment variable, but it needs to be done + // before the process starts. The following internal method allows us to do the same thing. + mono_set_assemblies_path(msbuildToolPath); + } +#endif + } + } +} diff --git a/src/csharp/Grpc.Tools.Tests/NUnitMain.cs b/src/csharp/Grpc.Tools.Tests/NUnitMain.cs index d30d608aa33..7e0c74898d3 100644 --- a/src/csharp/Grpc.Tools.Tests/NUnitMain.cs +++ b/src/csharp/Grpc.Tools.Tests/NUnitMain.cs @@ -23,7 +23,10 @@ namespace Grpc.Tools.Tests { static class NUnitMain { - public static int Main(string[] args) => - new AutoRun(typeof(NUnitMain).GetTypeInfo().Assembly).Execute(args); - }; + public static int Main(string[] args) + { + MsBuildAssemblyHelper.TweakAssemblyPathIfOnMono(); + return new AutoRun(typeof(NUnitMain).GetTypeInfo().Assembly).Execute(args); + } + } } From 7294172c8fd22801a27e548c5ab591a3e49a74a4 Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Wed, 10 Jun 2020 09:40:16 -0700 Subject: [PATCH 22/80] Avoid attribute error in __del__ from _ChannelCallState --- src/python/grpcio/grpc/_channel.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/python/grpcio/grpc/_channel.py b/src/python/grpcio/grpc/_channel.py index 8632ead740d..8bae8a12127 100644 --- a/src/python/grpcio/grpc/_channel.py +++ b/src/python/grpcio/grpc/_channel.py @@ -1123,7 +1123,9 @@ class _ChannelCallState(object): self.managed_calls = 0 def __del__(self): - self.channel.close(cygrpc.StatusCode.cancelled, 'Channel deallocated!') + if hasattr(self, 'channel') and self.channel: + self.channel.close(cygrpc.StatusCode.cancelled, + 'Channel deallocated!') def _run_channel_spin_thread(state): From 5727ccc28fc661fde8e18722cc78810a278e433b Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Wed, 10 Jun 2020 23:25:29 -0700 Subject: [PATCH 23/80] Rename deadline/timeout --- test/cpp/interop/client.cc | 6 +++--- test/cpp/interop/interop_client.cc | 14 +++++++------- test/cpp/interop/interop_client.h | 6 +++--- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/test/cpp/interop/client.cc b/test/cpp/interop/client.cc index a6c406a77ed..5ca6bd18621 100644 --- a/test/cpp/interop/client.cc +++ b/test/cpp/interop/client.cc @@ -97,7 +97,7 @@ DEFINE_int32(soak_max_failures, 0, DEFINE_int32(soak_per_iteration_max_acceptable_latency_ms, 0, "The number of milliseconds a single iteration in the two soak " "tests (rpc_soak and channel_soak) should take."); -DEFINE_int32(soak_overall_deadline_seconds, 0, +DEFINE_int32(soak_overall_timeout_seconds, 0, "The overall number of seconds after which a soak test should " "stop and fail, if the desired number of iterations have not yet " "completed."); @@ -270,12 +270,12 @@ int main(int argc, char** argv) { std::bind(&grpc::testing::InteropClient::DoChannelSoakTest, &client, FLAGS_soak_iterations, FLAGS_soak_max_failures, FLAGS_soak_per_iteration_max_acceptable_latency_ms, - FLAGS_soak_overall_deadline_seconds); + FLAGS_soak_overall_timeout_seconds); actions["rpc_soak"] = std::bind(&grpc::testing::InteropClient::DoRpcSoakTest, &client, FLAGS_soak_iterations, FLAGS_soak_max_failures, FLAGS_soak_per_iteration_max_acceptable_latency_ms, - FLAGS_soak_overall_deadline_seconds); + FLAGS_soak_overall_timeout_seconds); actions["long_lived_channel"] = std::bind(&grpc::testing::InteropClient::DoLongLivedChannelTest, &client, FLAGS_soak_iterations, FLAGS_iteration_interval); diff --git a/test/cpp/interop/interop_client.cc b/test/cpp/interop/interop_client.cc index 7c7f9fdd458..f0c65482fcf 100644 --- a/test/cpp/interop/interop_client.cc +++ b/test/cpp/interop/interop_client.cc @@ -1107,14 +1107,14 @@ void InteropClient::PerformSoakTest( const bool reset_channel_per_iteration, const int32_t soak_iterations, const int32_t max_failures, const int32_t max_acceptable_per_iteration_latency_ms, - const int32_t overall_deadline_seconds) { + const int32_t overall_timeout_seconds) { std::vector> results; grpc_histogram* latencies_ms_histogram = grpc_histogram_create( 1 /* resolution */, 500 * 1e3 /* largest bucket; 500 seconds is unlikely */); gpr_timespec overall_deadline = gpr_time_add( gpr_now(GPR_CLOCK_MONOTONIC), - gpr_time_from_seconds(overall_deadline_seconds, GPR_TIMESPAN)); + gpr_time_from_seconds(overall_timeout_seconds, GPR_TIMESPAN)); int32_t iterations_ran = 0; for (int i = 0; i < soak_iterations && @@ -1159,7 +1159,7 @@ void InteropClient::PerformSoakTest( "Some or all of the iterations that did run were unexpectedly slow. " "See breakdown above for which iterations succeeded, failed, and " "why for more info.", - overall_deadline_seconds, iterations_ran, soak_iterations, + overall_timeout_seconds, iterations_ran, soak_iterations, total_failures, max_failures, latency_ms_median, latency_ms_90th, latency_ms_worst); GPR_ASSERT(0); @@ -1192,12 +1192,12 @@ void InteropClient::PerformSoakTest( bool InteropClient::DoRpcSoakTest( int32_t soak_iterations, int32_t max_failures, int64_t max_acceptable_per_iteration_latency_ms, - int32_t overall_deadline_seconds) { + int32_t overall_timeout_seconds) { gpr_log(GPR_DEBUG, "Sending %d RPCs...", soak_iterations); GPR_ASSERT(soak_iterations > 0); PerformSoakTest(false /* reset channel per iteration */, soak_iterations, max_failures, max_acceptable_per_iteration_latency_ms, - overall_deadline_seconds); + overall_timeout_seconds); gpr_log(GPR_DEBUG, "rpc_soak test done."); return true; } @@ -1205,13 +1205,13 @@ bool InteropClient::DoRpcSoakTest( bool InteropClient::DoChannelSoakTest( int32_t soak_iterations, int32_t max_failures, int64_t max_acceptable_per_iteration_latency_ms, - int32_t overall_deadline_seconds) { + int32_t overall_timeout_seconds) { gpr_log(GPR_DEBUG, "Sending %d RPCs, tearing down the channel each time...", soak_iterations); GPR_ASSERT(soak_iterations > 0); PerformSoakTest(true /* reset channel per iteration */, soak_iterations, max_failures, max_acceptable_per_iteration_latency_ms, - overall_deadline_seconds); + overall_timeout_seconds); gpr_log(GPR_DEBUG, "channel_soak test done."); return true; } diff --git a/test/cpp/interop/interop_client.h b/test/cpp/interop/interop_client.h index 3d334f748fe..f9ecbd634b6 100644 --- a/test/cpp/interop/interop_client.h +++ b/test/cpp/interop/interop_client.h @@ -78,10 +78,10 @@ class InteropClient { // languages bool DoChannelSoakTest(int32_t soak_iterations, int32_t max_failures, int64_t max_acceptable_per_iteration_latency_ms, - int32_t overall_deadline_seconds); + int32_t overall_timeout_seconds); bool DoRpcSoakTest(int32_t soak_iterations, int32_t max_failures, int64_t max_acceptable_per_iteration_latency_ms, - int32_t overall_deadline_seconds); + int32_t overall_timeout_seconds); bool DoLongLivedChannelTest(int32_t soak_iterations, int32_t iteration_interval); @@ -140,7 +140,7 @@ class InteropClient { const int32_t soak_iterations, const int32_t max_failures, const int32_t max_acceptable_per_iteration_latency_ms, - const int32_t overall_deadline_seconds); + const int32_t overall_timeout_seconds); ServiceStub serviceStub_; /// If true, abort() is not called for transient failures From 376c0f0767058bd42dc67b53c2444b20401abfd1 Mon Sep 17 00:00:00 2001 From: Mariano Anaya Date: Thu, 11 Jun 2020 09:32:55 +0200 Subject: [PATCH 24/80] Add missing metadata TypeError case Code formatted. --- src/python/grpcio/grpc/experimental/aio/_base_call.py | 3 +-- src/python/grpcio/grpc/experimental/aio/_base_server.py | 6 ++---- src/python/grpcio/grpc/experimental/aio/_interceptor.py | 4 ++-- src/python/grpcio_tests/tests_aio/unit/metadata_test.py | 4 ++++ 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/python/grpcio/grpc/experimental/aio/_base_call.py b/src/python/grpcio/grpc/experimental/aio/_base_call.py index c07b4ca6b14..4ccbb3be132 100644 --- a/src/python/grpcio/grpc/experimental/aio/_base_call.py +++ b/src/python/grpcio/grpc/experimental/aio/_base_call.py @@ -23,8 +23,7 @@ from typing import AsyncIterable, Awaitable, Generic, Optional, Union import grpc -from ._typing import (DoneCallbackType, EOFType, RequestType, - ResponseType) +from ._typing import (DoneCallbackType, EOFType, RequestType, ResponseType) from ._metadata import Metadata __all__ = 'RpcContext', 'Call', 'UnaryUnaryCall', 'UnaryStreamCall' diff --git a/src/python/grpcio/grpc/experimental/aio/_base_server.py b/src/python/grpcio/grpc/experimental/aio/_base_server.py index 842e9b15c9e..86c15fc86b0 100644 --- a/src/python/grpcio/grpc/experimental/aio/_base_server.py +++ b/src/python/grpcio/grpc/experimental/aio/_base_server.py @@ -158,8 +158,7 @@ class ServicerContext(Generic[RequestType, ResponseType], abc.ABC): """ @abc.abstractmethod - async def send_initial_metadata(self, - initial_metadata: Metadata) -> None: + async def send_initial_metadata(self, initial_metadata: Metadata) -> None: """Sends the initial metadata value to the client. This method need not be called by implementations if they have no @@ -191,8 +190,7 @@ class ServicerContext(Generic[RequestType, ResponseType], abc.ABC): """ @abc.abstractmethod - async def set_trailing_metadata(self, - trailing_metadata: Metadata) -> None: + async def set_trailing_metadata(self, trailing_metadata: Metadata) -> None: """Sends the trailing metadata for the RPC. This method need not be called by implementations if they have no diff --git a/src/python/grpcio/grpc/experimental/aio/_interceptor.py b/src/python/grpcio/grpc/experimental/aio/_interceptor.py index c8f185afb56..80e9625c553 100644 --- a/src/python/grpcio/grpc/experimental/aio/_interceptor.py +++ b/src/python/grpcio/grpc/experimental/aio/_interceptor.py @@ -27,8 +27,8 @@ from ._call import _RPC_ALREADY_FINISHED_DETAILS, _RPC_HALF_CLOSED_DETAILS from ._call import _API_STYLE_ERROR from ._utils import _timeout_to_deadline from ._typing import (RequestType, SerializingFunction, DeserializingFunction, - ResponseType, DoneCallbackType, - RequestIterableType, ResponseIterableType) + ResponseType, DoneCallbackType, RequestIterableType, + ResponseIterableType) from ._metadata import Metadata _LOCAL_CANCELLATION_DETAILS = 'Locally cancelled by application!' diff --git a/src/python/grpcio_tests/tests_aio/unit/metadata_test.py b/src/python/grpcio_tests/tests_aio/unit/metadata_test.py index 0c8956537ce..c1fa97b3e4c 100644 --- a/src/python/grpcio_tests/tests_aio/unit/metadata_test.py +++ b/src/python/grpcio_tests/tests_aio/unit/metadata_test.py @@ -65,6 +65,10 @@ _INVALID_METADATA_TEST_CASES = ( TypeError, ((None, {}),), ), + ( + TypeError, + (({}, {}),), + ), ( TypeError, (('normal', object()),), From b269fdbf8b152eba950e50e658b239066780e9bb Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Thu, 11 Jun 2020 09:30:31 -0700 Subject: [PATCH 25/80] Don't mark server RPC canceled if finished with non-OK status --- CMakeLists.txt | 4 +- Makefile | 40 +++++++-------- build_config.rb | 2 +- build_handwritten.yaml | 2 +- include/grpc/impl/codegen/grpc_types.h | 6 ++- .../grpcpp/impl/codegen/server_context_impl.h | 18 +++++-- .../chttp2/transport/chttp2_transport.cc | 3 ++ .../ext/transport/chttp2/transport/internal.h | 7 +++ .../ext/transport/chttp2/transport/writing.cc | 4 ++ .../ext/transport/inproc/inproc_transport.cc | 3 ++ src/core/lib/surface/call.cc | 16 +++--- src/core/lib/surface/version.cc | 2 +- src/core/lib/transport/transport.h | 6 +++ src/objective-c/tests/version.h | 2 +- test/core/end2end/dualstack_socket_test.cc | 2 +- test/core/end2end/inproc_callback_test.cc | 2 +- test/core/end2end/tests/call_host_override.cc | 2 +- test/core/end2end/tests/default_host.cc | 2 +- .../core/end2end/tests/disappearing_server.cc | 2 +- .../end2end/tests/graceful_server_shutdown.cc | 2 +- test/core/end2end/tests/high_initial_seqno.cc | 2 +- test/core/end2end/tests/hpack_size.cc | 2 +- test/core/end2end/tests/idempotent_request.cc | 2 +- .../end2end/tests/invoke_large_request.cc | 2 +- .../end2end/tests/max_concurrent_streams.cc | 2 +- test/core/end2end/tests/max_connection_age.cc | 2 +- .../core/end2end/tests/max_connection_idle.cc | 2 +- test/core/end2end/tests/no_logging.cc | 2 +- test/core/end2end/tests/proxy_auth.cc | 2 +- test/core/end2end/tests/registered_call.cc | 2 +- test/core/end2end/tests/retry_cancellation.cc | 2 +- test/core/end2end/tests/retry_disabled.cc | 2 +- ...ry_exceeds_buffer_size_in_initial_batch.cc | 2 +- ...exceeds_buffer_size_in_subsequent_batch.cc | 2 +- .../tests/retry_non_retriable_status.cc | 2 +- ...s_before_recv_trailing_metadata_started.cc | 2 +- .../tests/retry_recv_initial_metadata.cc | 2 +- test/core/end2end/tests/retry_recv_message.cc | 2 +- .../tests/retry_server_pushback_disabled.cc | 2 +- test/core/end2end/tests/retry_streaming.cc | 2 +- .../tests/retry_streaming_after_commit.cc | 2 +- ...reaming_succeeds_before_replay_finished.cc | 2 +- test/core/end2end/tests/retry_throttled.cc | 2 +- .../end2end/tests/retry_too_many_attempts.cc | 2 +- .../end2end/tests/server_finishes_request.cc | 2 +- .../end2end/tests/simple_delayed_request.cc | 2 +- test/core/end2end/tests/simple_request.cc | 2 +- .../end2end/tests/streaming_error_response.cc | 2 +- test/cpp/end2end/async_end2end_test.cc | 51 ++++++++++++++++++- .../end2end/client_callback_end2end_test.cc | 35 +++++++++++++ test/cpp/end2end/test_service_impl.cc | 4 ++ tools/doxygen/Doxyfile.core | 2 +- tools/doxygen/Doxyfile.core.internal | 2 +- 53 files changed, 201 insertions(+), 76 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 323f7800627..158b3c03f34 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,8 +26,8 @@ cmake_minimum_required(VERSION 3.5.1) set(PACKAGE_NAME "grpc") set(PACKAGE_VERSION "1.31.0-dev") -set(gRPC_CORE_VERSION "10.0.0") -set(gRPC_CORE_SOVERSION "10") +set(gRPC_CORE_VERSION "11.0.0") +set(gRPC_CORE_SOVERSION "11") set(gRPC_CPP_VERSION "1.31.0-dev") set(gRPC_CPP_SOVERSION "1") set(gRPC_CSHARP_VERSION "2.31.0-dev") diff --git a/Makefile b/Makefile index bbaccc44513..80d3be9e53a 100644 --- a/Makefile +++ b/Makefile @@ -469,7 +469,7 @@ E = @echo Q = @ endif -CORE_VERSION = 10.0.0 +CORE_VERSION = 11.0.0 CPP_VERSION = 1.31.0-dev CSHARP_VERSION = 2.31.0-dev @@ -519,7 +519,7 @@ SHARED_EXT_CORE = dll SHARED_EXT_CPP = dll SHARED_EXT_CSHARP = dll SHARED_PREFIX = -SHARED_VERSION_CORE = -10 +SHARED_VERSION_CORE = -11 SHARED_VERSION_CPP = -1 SHARED_VERSION_CSHARP = -2 else ifeq ($(SYSTEM),Darwin) @@ -3042,7 +3042,7 @@ install-shared_c: shared_c strip-shared_c install-pkg-config_c ifeq ($(SYSTEM),MINGW32) $(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libaddress_sorting$(SHARED_VERSION_CORE)-dll.a $(prefix)/lib/libaddress_sorting.a else ifneq ($(SYSTEM),Darwin) - $(Q) ln -sf $(SHARED_PREFIX)address_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libaddress_sorting.so.10 + $(Q) ln -sf $(SHARED_PREFIX)address_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libaddress_sorting.so.11 $(Q) ln -sf $(SHARED_PREFIX)address_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libaddress_sorting.so endif $(E) "[INSTALL] Installing $(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE)" @@ -3051,7 +3051,7 @@ endif ifeq ($(SYSTEM),MINGW32) $(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE)-dll.a $(prefix)/lib/libgpr.a else ifneq ($(SYSTEM),Darwin) - $(Q) ln -sf $(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libgpr.so.10 + $(Q) ln -sf $(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libgpr.so.11 $(Q) ln -sf $(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libgpr.so endif $(E) "[INSTALL] Installing $(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE)" @@ -3060,7 +3060,7 @@ endif ifeq ($(SYSTEM),MINGW32) $(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE)-dll.a $(prefix)/lib/libgrpc.a else ifneq ($(SYSTEM),Darwin) - $(Q) ln -sf $(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libgrpc.so.10 + $(Q) ln -sf $(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libgrpc.so.11 $(Q) ln -sf $(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libgrpc.so endif $(E) "[INSTALL] Installing $(SHARED_PREFIX)grpc_csharp_ext$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE)" @@ -3069,7 +3069,7 @@ endif ifeq ($(SYSTEM),MINGW32) $(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgrpc_csharp_ext$(SHARED_VERSION_CORE)-dll.a $(prefix)/lib/libgrpc_csharp_ext.a else ifneq ($(SYSTEM),Darwin) - $(Q) ln -sf $(SHARED_PREFIX)grpc_csharp_ext$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libgrpc_csharp_ext.so.10 + $(Q) ln -sf $(SHARED_PREFIX)grpc_csharp_ext$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libgrpc_csharp_ext.so.11 $(Q) ln -sf $(SHARED_PREFIX)grpc_csharp_ext$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libgrpc_csharp_ext.so endif $(E) "[INSTALL] Installing $(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE)" @@ -3078,7 +3078,7 @@ endif ifeq ($(SYSTEM),MINGW32) $(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE)-dll.a $(prefix)/lib/libgrpc_unsecure.a else ifneq ($(SYSTEM),Darwin) - $(Q) ln -sf $(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libgrpc_unsecure.so.10 + $(Q) ln -sf $(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libgrpc_unsecure.so.11 $(Q) ln -sf $(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libgrpc_unsecure.so endif $(E) "[INSTALL] Installing $(SHARED_PREFIX)upb$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE)" @@ -3087,7 +3087,7 @@ endif ifeq ($(SYSTEM),MINGW32) $(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libupb$(SHARED_VERSION_CORE)-dll.a $(prefix)/lib/libupb.a else ifneq ($(SYSTEM),Darwin) - $(Q) ln -sf $(SHARED_PREFIX)upb$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libupb.so.10 + $(Q) ln -sf $(SHARED_PREFIX)upb$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libupb.so.11 $(Q) ln -sf $(SHARED_PREFIX)upb$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libupb.so endif ifneq ($(SYSTEM),MINGW32) @@ -3248,8 +3248,8 @@ $(LIBDIR)/$(CONFIG)/libaddress_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): ifeq ($(SYSTEM),Darwin) $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)address_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libaddress_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBADDRESS_SORTING_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(UPB_MERGE_LIBS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) else - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libaddress_sorting.so.10 -o $(LIBDIR)/$(CONFIG)/libaddress_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBADDRESS_SORTING_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(UPB_MERGE_LIBS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) - $(Q) ln -sf $(SHARED_PREFIX)address_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libaddress_sorting$(SHARED_VERSION_CORE).so.10 + $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libaddress_sorting.so.11 -o $(LIBDIR)/$(CONFIG)/libaddress_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBADDRESS_SORTING_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(UPB_MERGE_LIBS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) + $(Q) ln -sf $(SHARED_PREFIX)address_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libaddress_sorting$(SHARED_VERSION_CORE).so.11 $(Q) ln -sf $(SHARED_PREFIX)address_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libaddress_sorting$(SHARED_VERSION_CORE).so endif endif @@ -3610,8 +3610,8 @@ $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBGPR_OB ifeq ($(SYSTEM),Darwin) $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGPR_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(UPB_MERGE_LIBS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) else - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgpr.so.10 -o $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGPR_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(UPB_MERGE_LIBS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) - $(Q) ln -sf $(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).so.10 + $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgpr.so.11 -o $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGPR_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(UPB_MERGE_LIBS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) + $(Q) ln -sf $(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).so.11 $(Q) ln -sf $(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).so endif endif @@ -4092,8 +4092,8 @@ $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBGRPC_ ifeq ($(SYSTEM),Darwin) $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_OBJS) $(OPENSSL_MERGE_LIBS) $(LDLIBS_SECURE) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(UPB_MERGE_LIBS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) -lgpr -laddress_sorting -lupb else - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgrpc.so.10 -o $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_OBJS) $(OPENSSL_MERGE_LIBS) $(LDLIBS_SECURE) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(UPB_MERGE_LIBS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) -lgpr -laddress_sorting -lupb - $(Q) ln -sf $(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).so.10 + $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgrpc.so.11 -o $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_OBJS) $(OPENSSL_MERGE_LIBS) $(LDLIBS_SECURE) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(UPB_MERGE_LIBS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) -lgpr -laddress_sorting -lupb + $(Q) ln -sf $(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).so.11 $(Q) ln -sf $(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).so endif endif @@ -4149,8 +4149,8 @@ $(LIBDIR)/$(CONFIG)/libgrpc_csharp_ext$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): ifeq ($(SYSTEM),Darwin) $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)grpc_csharp_ext$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libgrpc_csharp_ext$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_CSHARP_EXT_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(UPB_MERGE_LIBS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) -lgrpc -lgpr -laddress_sorting -lupb else - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgrpc_csharp_ext.so.10 -o $(LIBDIR)/$(CONFIG)/libgrpc_csharp_ext$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_CSHARP_EXT_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(UPB_MERGE_LIBS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) -lgrpc -lgpr -laddress_sorting -lupb - $(Q) ln -sf $(SHARED_PREFIX)grpc_csharp_ext$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc_csharp_ext$(SHARED_VERSION_CORE).so.10 + $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgrpc_csharp_ext.so.11 -o $(LIBDIR)/$(CONFIG)/libgrpc_csharp_ext$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_CSHARP_EXT_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(UPB_MERGE_LIBS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) -lgrpc -lgpr -laddress_sorting -lupb + $(Q) ln -sf $(SHARED_PREFIX)grpc_csharp_ext$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc_csharp_ext$(SHARED_VERSION_CORE).so.11 $(Q) ln -sf $(SHARED_PREFIX)grpc_csharp_ext$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc_csharp_ext$(SHARED_VERSION_CORE).so endif endif @@ -4650,8 +4650,8 @@ $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $ ifeq ($(SYSTEM),Darwin) $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_UNSECURE_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(UPB_MERGE_LIBS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) -lgpr -laddress_sorting -lupb else - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgrpc_unsecure.so.10 -o $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_UNSECURE_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(UPB_MERGE_LIBS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) -lgpr -laddress_sorting -lupb - $(Q) ln -sf $(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).so.10 + $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgrpc_unsecure.so.11 -o $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_UNSECURE_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(UPB_MERGE_LIBS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) -lgpr -laddress_sorting -lupb + $(Q) ln -sf $(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).so.11 $(Q) ln -sf $(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).so endif endif @@ -6271,8 +6271,8 @@ $(LIBDIR)/$(CONFIG)/libupb$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBUPB_OB ifeq ($(SYSTEM),Darwin) $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)upb$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libupb$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUPB_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(UPB_MERGE_LIBS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) else - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libupb.so.10 -o $(LIBDIR)/$(CONFIG)/libupb$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUPB_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(UPB_MERGE_LIBS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) - $(Q) ln -sf $(SHARED_PREFIX)upb$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libupb$(SHARED_VERSION_CORE).so.10 + $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libupb.so.11 -o $(LIBDIR)/$(CONFIG)/libupb$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUPB_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(UPB_MERGE_LIBS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) + $(Q) ln -sf $(SHARED_PREFIX)upb$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libupb$(SHARED_VERSION_CORE).so.11 $(Q) ln -sf $(SHARED_PREFIX)upb$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libupb$(SHARED_VERSION_CORE).so endif endif diff --git a/build_config.rb b/build_config.rb index bbbf04812b2..ff432bd3392 100644 --- a/build_config.rb +++ b/build_config.rb @@ -13,5 +13,5 @@ # limitations under the License. module GrpcBuildConfig - CORE_WINDOWS_DLL = '/tmp/libs/opt/grpc-10.dll' + CORE_WINDOWS_DLL = '/tmp/libs/opt/grpc-11.dll' end diff --git a/build_handwritten.yaml b/build_handwritten.yaml index f1e603ababb..15594c622da 100644 --- a/build_handwritten.yaml +++ b/build_handwritten.yaml @@ -12,7 +12,7 @@ settings: '#08': Use "-preN" suffixes to identify pre-release versions '#09': Per-language overrides are possible with (eg) ruby_version tag here '#10': See the expand_version.py for all the quirks here - core_version: 10.0.0 + core_version: 11.0.0 csharp_major_version: 2 g_stands_for: galore version: 1.31.0-dev diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h index 725ecd244d2..aeb8f9c3ce1 100644 --- a/include/grpc/impl/codegen/grpc_types.h +++ b/include/grpc/impl/codegen/grpc_types.h @@ -674,8 +674,10 @@ typedef struct grpc_op { const char** error_string; } recv_status_on_client; struct grpc_op_recv_close_on_server { - /** out argument, set to 1 if the call failed in any way (seen as a - cancellation on the server), or 0 if the call succeeded */ + /** out argument, set to 1 if the call failed at the server for + a reason other than a non-OK status (cancel, deadline + exceeded, network failure, etc.), 0 otherwise (RPC processing ran to + completion and was able to provide any status from the server) */ int* cancelled; } recv_close_on_server; } data; diff --git a/include/grpcpp/impl/codegen/server_context_impl.h b/include/grpcpp/impl/codegen/server_context_impl.h index c5b874aa351..77d1c026b5d 100644 --- a/include/grpcpp/impl/codegen/server_context_impl.h +++ b/include/grpcpp/impl/codegen/server_context_impl.h @@ -174,6 +174,14 @@ class ServerContextBase { /// ASCII-Value -> 1*( %x20-%x7E ) ; space and printable ASCII void AddTrailingMetadata(const grpc::string& key, const grpc::string& value); + /// Return whether this RPC failed before the server could provide its status + /// back to the client. This could be because of explicit API cancellation + /// from the client-side or server-side, because of deadline exceeded, network + /// connection reset, HTTP/2 parameter configuration (e.g., max message size, + /// max connection age), etc. It does NOT include failure due to a non-OK + /// status return from the server application's request handler, including + /// Status::CANCELLED. + /// /// IsCancelled is always safe to call when using sync or callback API. /// When using async API, it is only safe to call IsCancelled after /// the AsyncNotifyWhenDone tag has been delivered. Thread-safe. @@ -181,10 +189,9 @@ class ServerContextBase { /// Cancel the Call from the server. This is a best-effort API and /// depending on when it is called, the RPC may still appear successful to - /// the client. - /// For example, if TryCancel() is called on a separate thread, it might race - /// with the server handler which might return success to the client before - /// TryCancel() was even started by the thread. + /// the client. For example, if TryCancel() is called on a separate thread, it + /// might race with the server handler which might return success to the + /// client before TryCancel() was even started by the thread. /// /// It is the caller's responsibility to prevent such races and ensure that if /// TryCancel() is called, the serverhandler must return Status::CANCELLED. @@ -192,6 +199,9 @@ class ServerContextBase { /// error status code, it is ok to not return Status::CANCELLED even if /// TryCancel() was called. /// + /// For reasons such as the above, it is generally preferred to explicitly + /// finish an RPC by returning Status::CANCELLED rather than using TryCancel. + /// /// Note that TryCancel() does not change any of the tags that are pending /// on the completion queue. All pending tags will still be delivered /// (though their ok result may reflect the effect of cancellation). diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc index 24c9d38417b..efe104aa0e2 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc @@ -1525,6 +1525,7 @@ static void perform_stream_op_locked(void* stream_op, s->send_trailing_metadata_finished = add_closure_barrier(on_complete); s->send_trailing_metadata = op_payload->send_trailing_metadata.send_trailing_metadata; + s->sent_trailing_metadata_op = op_payload->send_trailing_metadata.sent; s->write_buffering = false; const size_t metadata_size = grpc_metadata_batch_size(s->send_trailing_metadata); @@ -1550,6 +1551,7 @@ static void perform_stream_op_locked(void* stream_op, } if (s->write_closed) { s->send_trailing_metadata = nullptr; + s->sent_trailing_metadata_op = nullptr; grpc_chttp2_complete_closure_step( t, s, &s->send_trailing_metadata_finished, grpc_metadata_batch_is_empty( @@ -2185,6 +2187,7 @@ void grpc_chttp2_fail_pending_writes(grpc_chttp2_transport* t, "send_initial_metadata_finished"); s->send_trailing_metadata = nullptr; + s->sent_trailing_metadata_op = nullptr; grpc_chttp2_complete_closure_step(t, s, &s->send_trailing_metadata_finished, GRPC_ERROR_REF(error), "send_trailing_metadata_finished"); diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index 3dace804f5e..34589e0e340 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -527,6 +527,13 @@ struct grpc_chttp2_stream { grpc_metadata_batch* send_initial_metadata = nullptr; grpc_closure* send_initial_metadata_finished = nullptr; grpc_metadata_batch* send_trailing_metadata = nullptr; + // TODO(yashykt): Find a better name for the below field and others in this + // struct to betteer distinguish inputs, return values, and + // internal state. + // sent_trailing_metadata_op allows the transport to fill in to the upper + // layer whether this stream was able to send its trailing metadata (used for + // detecting cancellation on the server-side).. + bool* sent_trailing_metadata_op = nullptr; grpc_closure* send_trailing_metadata_finished = nullptr; grpc_core::OrphanablePtr fetching_send_message; diff --git a/src/core/ext/transport/chttp2/transport/writing.cc b/src/core/ext/transport/chttp2/transport/writing.cc index 3c61f8914a2..5948ac98f96 100644 --- a/src/core/ext/transport/chttp2/transport/writing.cc +++ b/src/core/ext/transport/chttp2/transport/writing.cc @@ -606,6 +606,10 @@ class StreamWriteContext { void SentLastFrame() { s_->send_trailing_metadata = nullptr; + if (s_->sent_trailing_metadata_op) { + *s_->sent_trailing_metadata_op = true; + s_->sent_trailing_metadata_op = nullptr; + } s_->sent_trailing_metadata = true; s_->eos_sent = true; diff --git a/src/core/ext/transport/inproc/inproc_transport.cc b/src/core/ext/transport/inproc/inproc_transport.cc index 424a37c3303..9370a441a4b 100644 --- a/src/core/ext/transport/inproc/inproc_transport.cc +++ b/src/core/ext/transport/inproc/inproc_transport.cc @@ -652,6 +652,9 @@ void op_state_machine_locked(inproc_stream* s, grpc_error* error) { 0, dest, nullptr, destfilled); } s->trailing_md_sent = true; + if (s->send_trailing_md_op->payload->send_trailing_metadata.sent) { + *s->send_trailing_md_op->payload->send_trailing_metadata.sent = true; + } if (!s->t->is_client && s->trailing_md_recvd && s->recv_trailing_md_op) { INPROC_LOG(GPR_INFO, "op_state_machine %p scheduling trailing-metadata-ready", s); diff --git a/src/core/lib/surface/call.cc b/src/core/lib/surface/call.cc index 81154b85e8e..f8848011a76 100644 --- a/src/core/lib/surface/call.cc +++ b/src/core/lib/surface/call.cc @@ -225,7 +225,9 @@ struct grpc_call { grpc_closure receiving_initial_metadata_ready; grpc_closure receiving_trailing_metadata_ready; uint32_t test_only_last_message_flags = 0; - gpr_atm cancelled = 0; + // Status about operation of call + bool sent_server_trailing_metadata = false; + gpr_atm cancelled_with_error = 0; grpc_closure release_call; @@ -686,7 +688,7 @@ static void done_termination(void* arg, grpc_error* /*error*/) { } static void cancel_with_error(grpc_call* c, grpc_error* error) { - if (!gpr_atm_rel_cas(&c->cancelled, 0, 1)) { + if (!gpr_atm_rel_cas(&c->cancelled_with_error, 0, 1)) { GRPC_ERROR_UNREF(error); return; } @@ -751,13 +753,13 @@ static void set_final_status(grpc_call* call, grpc_error* error) { } } else { *call->final_op.server.cancelled = - error != GRPC_ERROR_NONE || - reinterpret_cast(gpr_atm_acq_load(&call->status_error)) != - GRPC_ERROR_NONE; + error != GRPC_ERROR_NONE || !call->sent_server_trailing_metadata; grpc_core::channelz::ServerNode* channelz_server = grpc_server_get_channelz_node(call->final_op.server.server); if (channelz_server != nullptr) { - if (*call->final_op.server.cancelled) { + if (*call->final_op.server.cancelled || + reinterpret_cast( + gpr_atm_acq_load(&call->status_error)) != GRPC_ERROR_NONE) { channelz_server->RecordCallFailed(); } else { channelz_server->RecordCallSucceeded(); @@ -1791,6 +1793,8 @@ static grpc_call_error call_start_batch(grpc_call* call, const grpc_op* ops, } stream_op_payload->send_trailing_metadata.send_trailing_metadata = &call->metadata_batch[0 /* is_receiving */][1 /* is_trailing */]; + stream_op_payload->send_trailing_metadata.sent = + &call->sent_server_trailing_metadata; has_send_ops = true; break; } diff --git a/src/core/lib/surface/version.cc b/src/core/lib/surface/version.cc index a2cb5b7ad10..c5e3bff762b 100644 --- a/src/core/lib/surface/version.cc +++ b/src/core/lib/surface/version.cc @@ -23,6 +23,6 @@ #include -const char* grpc_version_string(void) { return "10.0.0"; } +const char* grpc_version_string(void) { return "11.0.0"; } const char* grpc_g_stands_for(void) { return "galore"; } diff --git a/src/core/lib/transport/transport.h b/src/core/lib/transport/transport.h index ab512896404..cb5b7f7758c 100644 --- a/src/core/lib/transport/transport.h +++ b/src/core/lib/transport/transport.h @@ -242,6 +242,12 @@ struct grpc_transport_stream_op_batch_payload { struct { grpc_metadata_batch* send_trailing_metadata = nullptr; + // Set by the transport to true if the stream successfully wrote the + // trailing metadata. If this is not set but there was a send trailing + // metadata op present, this can indicate that a server call can be marked + // as a cancellation (since the stream was write-closed before status could + // be delivered). + bool* sent = nullptr; } send_trailing_metadata; struct { diff --git a/src/objective-c/tests/version.h b/src/objective-c/tests/version.h index f793ca0c6a1..2aa4deaf1b1 100644 --- a/src/objective-c/tests/version.h +++ b/src/objective-c/tests/version.h @@ -23,4 +23,4 @@ // `tools/buildgen/generate_projects.sh`. #define GRPC_OBJC_VERSION_STRING @"1.31.0-dev" -#define GRPC_C_VERSION_STRING @"10.0.0" +#define GRPC_C_VERSION_STRING @"11.0.0" diff --git a/test/core/end2end/dualstack_socket_test.cc b/test/core/end2end/dualstack_socket_test.cc index affe6d973f8..d13639e5062 100644 --- a/test/core/end2end/dualstack_socket_test.cc +++ b/test/core/end2end/dualstack_socket_test.cc @@ -232,7 +232,7 @@ void test_connect(const char* server_host, const char* client_host, int port, GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/foo")); GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.host, "foo.test.google.fr")); - GPR_ASSERT(was_cancelled == 1); + GPR_ASSERT(was_cancelled == 0); grpc_call_unref(s); } else { diff --git a/test/core/end2end/inproc_callback_test.cc b/test/core/end2end/inproc_callback_test.cc index 60f2348c9df..8605a6a409b 100644 --- a/test/core/end2end/inproc_callback_test.cc +++ b/test/core/end2end/inproc_callback_test.cc @@ -422,7 +422,7 @@ static void simple_request_body(grpc_end2end_test_config config, GPR_ASSERT(nullptr != strstr(error_string, "grpc_status")); GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/foo")); GPR_ASSERT(0 == call_details.flags); - GPR_ASSERT(was_cancelled == 1); + GPR_ASSERT(was_cancelled == 0); grpc_slice_unref(details); gpr_free(static_cast(const_cast(error_string))); diff --git a/test/core/end2end/tests/call_host_override.cc b/test/core/end2end/tests/call_host_override.cc index 251dc6d045a..b5b79fda705 100644 --- a/test/core/end2end/tests/call_host_override.cc +++ b/test/core/end2end/tests/call_host_override.cc @@ -206,7 +206,7 @@ static void test_invoke_simple_request(grpc_end2end_test_config config) { GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/foo")); validate_host_override_string("foo.test.google.fr:1234", call_details.host, config); - GPR_ASSERT(was_cancelled == 1); + GPR_ASSERT(was_cancelled == 0); grpc_slice_unref(details); grpc_metadata_array_destroy(&initial_metadata_recv); diff --git a/test/core/end2end/tests/default_host.cc b/test/core/end2end/tests/default_host.cc index 22c3102d1a2..ec2baef4c96 100644 --- a/test/core/end2end/tests/default_host.cc +++ b/test/core/end2end/tests/default_host.cc @@ -201,7 +201,7 @@ static void test_invoke_simple_request(grpc_end2end_test_config config) { GPR_ASSERT(grpc_slice_buf_start_eq(call_details.host, "localhost", 9) || grpc_slice_buf_start_eq(call_details.host, "127.0.0.1", 9)); } - GPR_ASSERT(was_cancelled == 1); + GPR_ASSERT(was_cancelled == 0); grpc_slice_unref(details); grpc_metadata_array_destroy(&initial_metadata_recv); diff --git a/test/core/end2end/tests/disappearing_server.cc b/test/core/end2end/tests/disappearing_server.cc index 280ae7dd00a..b281d44003d 100644 --- a/test/core/end2end/tests/disappearing_server.cc +++ b/test/core/end2end/tests/disappearing_server.cc @@ -173,7 +173,7 @@ static void do_request_and_shutdown_server(grpc_end2end_test_config /*config*/, GPR_ASSERT(status == GRPC_STATUS_UNIMPLEMENTED); GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz")); GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/foo")); - GPR_ASSERT(was_cancelled == 1); + GPR_ASSERT(was_cancelled == 0); grpc_slice_unref(details); grpc_metadata_array_destroy(&initial_metadata_recv); diff --git a/test/core/end2end/tests/graceful_server_shutdown.cc b/test/core/end2end/tests/graceful_server_shutdown.cc index 42f2d1a4146..376b00bca24 100644 --- a/test/core/end2end/tests/graceful_server_shutdown.cc +++ b/test/core/end2end/tests/graceful_server_shutdown.cc @@ -181,7 +181,7 @@ static void test_early_server_shutdown_finishes_inflight_calls( GPR_ASSERT(status == GRPC_STATUS_UNIMPLEMENTED); GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/foo")); - GPR_ASSERT(was_cancelled == 1); + GPR_ASSERT(was_cancelled == 0); grpc_slice_unref(details); grpc_metadata_array_destroy(&initial_metadata_recv); diff --git a/test/core/end2end/tests/high_initial_seqno.cc b/test/core/end2end/tests/high_initial_seqno.cc index ca6e883ee4d..7db5c63b0f8 100644 --- a/test/core/end2end/tests/high_initial_seqno.cc +++ b/test/core/end2end/tests/high_initial_seqno.cc @@ -179,7 +179,7 @@ static void simple_request_body(grpc_end2end_test_config /*config*/, GPR_ASSERT(status == GRPC_STATUS_UNIMPLEMENTED); GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz")); GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/foo")); - GPR_ASSERT(was_cancelled == 1); + GPR_ASSERT(was_cancelled == 0); grpc_slice_unref(details); grpc_metadata_array_destroy(&initial_metadata_recv); diff --git a/test/core/end2end/tests/hpack_size.cc b/test/core/end2end/tests/hpack_size.cc index 8f36868d98a..e9abf1420b8 100644 --- a/test/core/end2end/tests/hpack_size.cc +++ b/test/core/end2end/tests/hpack_size.cc @@ -333,7 +333,7 @@ static void simple_request_body(grpc_end2end_test_config /*config*/, GPR_ASSERT(status == GRPC_STATUS_UNIMPLEMENTED); GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz")); GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/foo")); - GPR_ASSERT(was_cancelled == 1); + GPR_ASSERT(was_cancelled == 0); grpc_slice_unref(details); grpc_metadata_array_destroy(&initial_metadata_recv); diff --git a/test/core/end2end/tests/idempotent_request.cc b/test/core/end2end/tests/idempotent_request.cc index 5fc087c9e64..f91375c0ca6 100644 --- a/test/core/end2end/tests/idempotent_request.cc +++ b/test/core/end2end/tests/idempotent_request.cc @@ -194,7 +194,7 @@ static void simple_request_body(grpc_end2end_test_config /*config*/, GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz")); GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/foo")); GPR_ASSERT(GRPC_INITIAL_METADATA_IDEMPOTENT_REQUEST == call_details.flags); - GPR_ASSERT(was_cancelled == 1); + GPR_ASSERT(was_cancelled == 0); grpc_slice_unref(details); grpc_metadata_array_destroy(&initial_metadata_recv); diff --git a/test/core/end2end/tests/invoke_large_request.cc b/test/core/end2end/tests/invoke_large_request.cc index 39d90ab64df..cf25575ab23 100644 --- a/test/core/end2end/tests/invoke_large_request.cc +++ b/test/core/end2end/tests/invoke_large_request.cc @@ -235,7 +235,7 @@ static void test_invoke_large_request(grpc_end2end_test_config config, GPR_ASSERT(status == GRPC_STATUS_UNIMPLEMENTED); GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz")); GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/foo")); - GPR_ASSERT(was_cancelled == 1); + GPR_ASSERT(was_cancelled == 0); grpc_slice_unref(details); grpc_metadata_array_destroy(&initial_metadata_recv); diff --git a/test/core/end2end/tests/max_concurrent_streams.cc b/test/core/end2end/tests/max_concurrent_streams.cc index b3cc7af02ba..90f2f5a065d 100644 --- a/test/core/end2end/tests/max_concurrent_streams.cc +++ b/test/core/end2end/tests/max_concurrent_streams.cc @@ -175,7 +175,7 @@ static void simple_request_body(grpc_end2end_test_config /*config*/, GPR_ASSERT(status == GRPC_STATUS_UNIMPLEMENTED); GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz")); GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/foo")); - GPR_ASSERT(was_cancelled == 1); + GPR_ASSERT(was_cancelled == 0); grpc_slice_unref(details); grpc_metadata_array_destroy(&initial_metadata_recv); diff --git a/test/core/end2end/tests/max_connection_age.cc b/test/core/end2end/tests/max_connection_age.cc index 50c790d8521..a8979f0a5cf 100644 --- a/test/core/end2end/tests/max_connection_age.cc +++ b/test/core/end2end/tests/max_connection_age.cc @@ -342,7 +342,7 @@ static void test_max_age_gracefully_close(grpc_end2end_test_config config) { GPR_ASSERT(status == GRPC_STATUS_UNIMPLEMENTED); GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz")); GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/foo")); - GPR_ASSERT(was_cancelled == 1); + GPR_ASSERT(was_cancelled == 0); grpc_slice_unref(details); grpc_metadata_array_destroy(&initial_metadata_recv); diff --git a/test/core/end2end/tests/max_connection_idle.cc b/test/core/end2end/tests/max_connection_idle.cc index 87cbeff049d..53701ab6137 100644 --- a/test/core/end2end/tests/max_connection_idle.cc +++ b/test/core/end2end/tests/max_connection_idle.cc @@ -150,7 +150,7 @@ static void simple_request_body(grpc_end2end_test_config /*config*/, GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz")); GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/foo")); GPR_ASSERT(0 == call_details.flags); - GPR_ASSERT(was_cancelled == 1); + GPR_ASSERT(was_cancelled == 0); grpc_slice_unref(details); grpc_metadata_array_destroy(&initial_metadata_recv); diff --git a/test/core/end2end/tests/no_logging.cc b/test/core/end2end/tests/no_logging.cc index 34fbb01a2e9..2b92e4fa949 100644 --- a/test/core/end2end/tests/no_logging.cc +++ b/test/core/end2end/tests/no_logging.cc @@ -218,7 +218,7 @@ static void simple_request_body(grpc_end2end_test_config /*config*/, GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz")); GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/foo")); GPR_ASSERT(0 == call_details.flags); - GPR_ASSERT(was_cancelled == 1); + GPR_ASSERT(was_cancelled == 0); grpc_slice_unref(details); grpc_metadata_array_destroy(&initial_metadata_recv); diff --git a/test/core/end2end/tests/proxy_auth.cc b/test/core/end2end/tests/proxy_auth.cc index 3e9d7a3388d..3f1443f5b56 100644 --- a/test/core/end2end/tests/proxy_auth.cc +++ b/test/core/end2end/tests/proxy_auth.cc @@ -198,7 +198,7 @@ static void simple_request_body(grpc_end2end_test_config /*config*/, GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz")); GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/foo")); GPR_ASSERT(0 == call_details.flags); - GPR_ASSERT(was_cancelled == 1); + GPR_ASSERT(was_cancelled == 0); grpc_slice_unref(details); grpc_metadata_array_destroy(&initial_metadata_recv); diff --git a/test/core/end2end/tests/registered_call.cc b/test/core/end2end/tests/registered_call.cc index badd802335b..6228476e5e9 100644 --- a/test/core/end2end/tests/registered_call.cc +++ b/test/core/end2end/tests/registered_call.cc @@ -177,7 +177,7 @@ static void simple_request_body(grpc_end2end_test_config /*config*/, GPR_ASSERT(status == GRPC_STATUS_UNIMPLEMENTED); GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz")); GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/foo")); - GPR_ASSERT(was_cancelled == 1); + GPR_ASSERT(was_cancelled == 0); grpc_slice_unref(details); grpc_metadata_array_destroy(&initial_metadata_recv); diff --git a/test/core/end2end/tests/retry_cancellation.cc b/test/core/end2end/tests/retry_cancellation.cc index e764fe70a60..f6150f292ff 100644 --- a/test/core/end2end/tests/retry_cancellation.cc +++ b/test/core/end2end/tests/retry_cancellation.cc @@ -244,7 +244,7 @@ static void test_retry_cancellation(grpc_end2end_test_config config, cq_verify(cqv); GPR_ASSERT(status == mode.expect_status); - GPR_ASSERT(was_cancelled == 1); + GPR_ASSERT(was_cancelled == 0); grpc_slice_unref(details); grpc_metadata_array_destroy(&initial_metadata_recv); diff --git a/test/core/end2end/tests/retry_disabled.cc b/test/core/end2end/tests/retry_disabled.cc index ed3535409fe..340daed27b4 100644 --- a/test/core/end2end/tests/retry_disabled.cc +++ b/test/core/end2end/tests/retry_disabled.cc @@ -229,7 +229,7 @@ static void test_retry_disabled(grpc_end2end_test_config config) { GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz")); GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/service/method")); GPR_ASSERT(0 == call_details.flags); - GPR_ASSERT(was_cancelled == 1); + GPR_ASSERT(was_cancelled == 0); grpc_slice_unref(details); grpc_metadata_array_destroy(&initial_metadata_recv); diff --git a/test/core/end2end/tests/retry_exceeds_buffer_size_in_initial_batch.cc b/test/core/end2end/tests/retry_exceeds_buffer_size_in_initial_batch.cc index a033a0aa957..8d2dcc72ca4 100644 --- a/test/core/end2end/tests/retry_exceeds_buffer_size_in_initial_batch.cc +++ b/test/core/end2end/tests/retry_exceeds_buffer_size_in_initial_batch.cc @@ -232,7 +232,7 @@ static void test_retry_exceeds_buffer_size_in_initial_batch( GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz")); GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/service/method")); GPR_ASSERT(0 == call_details.flags); - GPR_ASSERT(was_cancelled == 1); + GPR_ASSERT(was_cancelled == 0); grpc_slice_unref(details); grpc_metadata_array_destroy(&initial_metadata_recv); diff --git a/test/core/end2end/tests/retry_exceeds_buffer_size_in_subsequent_batch.cc b/test/core/end2end/tests/retry_exceeds_buffer_size_in_subsequent_batch.cc index c1070e61918..6a93834dcdd 100644 --- a/test/core/end2end/tests/retry_exceeds_buffer_size_in_subsequent_batch.cc +++ b/test/core/end2end/tests/retry_exceeds_buffer_size_in_subsequent_batch.cc @@ -244,7 +244,7 @@ static void test_retry_exceeds_buffer_size_in_subsequent_batch( GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz")); GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/service/method")); GPR_ASSERT(0 == call_details.flags); - GPR_ASSERT(was_cancelled == 1); + GPR_ASSERT(was_cancelled == 0); grpc_slice_unref(details); grpc_metadata_array_destroy(&initial_metadata_recv); diff --git a/test/core/end2end/tests/retry_non_retriable_status.cc b/test/core/end2end/tests/retry_non_retriable_status.cc index b8d094749a5..b8ae4aded27 100644 --- a/test/core/end2end/tests/retry_non_retriable_status.cc +++ b/test/core/end2end/tests/retry_non_retriable_status.cc @@ -224,7 +224,7 @@ static void test_retry_non_retriable_status(grpc_end2end_test_config config) { GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz")); GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/service/method")); GPR_ASSERT(0 == call_details.flags); - GPR_ASSERT(was_cancelled == 1); + GPR_ASSERT(was_cancelled == 0); grpc_slice_unref(details); grpc_metadata_array_destroy(&initial_metadata_recv); diff --git a/test/core/end2end/tests/retry_non_retriable_status_before_recv_trailing_metadata_started.cc b/test/core/end2end/tests/retry_non_retriable_status_before_recv_trailing_metadata_started.cc index eb016a3de98..c0f35d98f2b 100644 --- a/test/core/end2end/tests/retry_non_retriable_status_before_recv_trailing_metadata_started.cc +++ b/test/core/end2end/tests/retry_non_retriable_status_before_recv_trailing_metadata_started.cc @@ -235,7 +235,7 @@ test_retry_non_retriable_status_before_recv_trailing_metadata_started( GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz")); GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/service/method")); GPR_ASSERT(0 == call_details.flags); - GPR_ASSERT(was_cancelled == 1); + GPR_ASSERT(was_cancelled == 0); grpc_slice_unref(details); grpc_metadata_array_destroy(&initial_metadata_recv); diff --git a/test/core/end2end/tests/retry_recv_initial_metadata.cc b/test/core/end2end/tests/retry_recv_initial_metadata.cc index 839b8703677..6205373c3ab 100644 --- a/test/core/end2end/tests/retry_recv_initial_metadata.cc +++ b/test/core/end2end/tests/retry_recv_initial_metadata.cc @@ -235,7 +235,7 @@ static void test_retry_recv_initial_metadata(grpc_end2end_test_config config) { GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz")); GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/service/method")); GPR_ASSERT(0 == call_details.flags); - GPR_ASSERT(was_cancelled == 1); + GPR_ASSERT(was_cancelled == 0); grpc_slice_unref(details); grpc_metadata_array_destroy(&initial_metadata_recv); diff --git a/test/core/end2end/tests/retry_recv_message.cc b/test/core/end2end/tests/retry_recv_message.cc index 5fdaad0b880..75be2588bf8 100644 --- a/test/core/end2end/tests/retry_recv_message.cc +++ b/test/core/end2end/tests/retry_recv_message.cc @@ -228,7 +228,7 @@ static void test_retry_recv_message(grpc_end2end_test_config config) { GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz")); GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/service/method")); GPR_ASSERT(0 == call_details.flags); - GPR_ASSERT(was_cancelled == 1); + GPR_ASSERT(was_cancelled == 0); grpc_slice_unref(details); grpc_metadata_array_destroy(&initial_metadata_recv); diff --git a/test/core/end2end/tests/retry_server_pushback_disabled.cc b/test/core/end2end/tests/retry_server_pushback_disabled.cc index 1c564761835..28515eb4d7f 100644 --- a/test/core/end2end/tests/retry_server_pushback_disabled.cc +++ b/test/core/end2end/tests/retry_server_pushback_disabled.cc @@ -273,7 +273,7 @@ static void test_retry_server_pushback_disabled( GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz")); GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/service/method")); GPR_ASSERT(0 == call_details.flags); - GPR_ASSERT(was_cancelled == 1); + GPR_ASSERT(was_cancelled == 0); grpc_slice_unref(details); grpc_metadata_array_destroy(&initial_metadata_recv); diff --git a/test/core/end2end/tests/retry_streaming.cc b/test/core/end2end/tests/retry_streaming.cc index e2171379e91..784014cf330 100644 --- a/test/core/end2end/tests/retry_streaming.cc +++ b/test/core/end2end/tests/retry_streaming.cc @@ -393,7 +393,7 @@ static void test_retry_streaming(grpc_end2end_test_config config) { GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz")); GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/service/method")); GPR_ASSERT(0 == call_details.flags); - GPR_ASSERT(was_cancelled == 1); + GPR_ASSERT(was_cancelled == 0); GPR_ASSERT(channelz_channel != nullptr); std::string json = channelz_channel->RenderJsonString(); diff --git a/test/core/end2end/tests/retry_streaming_after_commit.cc b/test/core/end2end/tests/retry_streaming_after_commit.cc index 05025d0a10b..509d56df238 100644 --- a/test/core/end2end/tests/retry_streaming_after_commit.cc +++ b/test/core/end2end/tests/retry_streaming_after_commit.cc @@ -311,7 +311,7 @@ static void test_retry_streaming_after_commit(grpc_end2end_test_config config) { GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz")); GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/service/method")); GPR_ASSERT(0 == call_details.flags); - GPR_ASSERT(was_cancelled == 1); + GPR_ASSERT(was_cancelled == 0); grpc_slice_unref(details); grpc_metadata_array_destroy(&initial_metadata_recv); diff --git a/test/core/end2end/tests/retry_streaming_succeeds_before_replay_finished.cc b/test/core/end2end/tests/retry_streaming_succeeds_before_replay_finished.cc index 14460d2a465..dbe2ef19f2d 100644 --- a/test/core/end2end/tests/retry_streaming_succeeds_before_replay_finished.cc +++ b/test/core/end2end/tests/retry_streaming_succeeds_before_replay_finished.cc @@ -363,7 +363,7 @@ static void test_retry_streaming_succeeds_before_replay_finished( GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz")); GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/service/method")); GPR_ASSERT(0 == call_details.flags); - GPR_ASSERT(was_cancelled == 1); + GPR_ASSERT(was_cancelled == 0); grpc_slice_unref(details); grpc_metadata_array_destroy(&initial_metadata_recv); diff --git a/test/core/end2end/tests/retry_throttled.cc b/test/core/end2end/tests/retry_throttled.cc index 0e286c3d17c..61ced017c14 100644 --- a/test/core/end2end/tests/retry_throttled.cc +++ b/test/core/end2end/tests/retry_throttled.cc @@ -231,7 +231,7 @@ static void test_retry_throttled(grpc_end2end_test_config config) { GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz")); GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/service/method")); GPR_ASSERT(0 == call_details.flags); - GPR_ASSERT(was_cancelled == 1); + GPR_ASSERT(was_cancelled == 0); grpc_slice_unref(details); grpc_metadata_array_destroy(&initial_metadata_recv); diff --git a/test/core/end2end/tests/retry_too_many_attempts.cc b/test/core/end2end/tests/retry_too_many_attempts.cc index 2af32679f1e..20944511f38 100644 --- a/test/core/end2end/tests/retry_too_many_attempts.cc +++ b/test/core/end2end/tests/retry_too_many_attempts.cc @@ -266,7 +266,7 @@ static void test_retry_too_many_attempts(grpc_end2end_test_config config) { GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz")); GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/service/method")); GPR_ASSERT(0 == call_details.flags); - GPR_ASSERT(was_cancelled == 1); + GPR_ASSERT(was_cancelled == 0); grpc_slice_unref(details); grpc_metadata_array_destroy(&initial_metadata_recv); diff --git a/test/core/end2end/tests/server_finishes_request.cc b/test/core/end2end/tests/server_finishes_request.cc index 9ee23c91ea0..5e7f3d96b4a 100644 --- a/test/core/end2end/tests/server_finishes_request.cc +++ b/test/core/end2end/tests/server_finishes_request.cc @@ -173,7 +173,7 @@ static void simple_request_body(grpc_end2end_test_config /*config*/, GPR_ASSERT(status == GRPC_STATUS_UNIMPLEMENTED); GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz")); GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/foo")); - GPR_ASSERT(was_cancelled == 1); + GPR_ASSERT(was_cancelled == 0); grpc_slice_unref(details); grpc_metadata_array_destroy(&initial_metadata_recv); diff --git a/test/core/end2end/tests/simple_delayed_request.cc b/test/core/end2end/tests/simple_delayed_request.cc index dc4fdbd6e81..4b2b7ce77fc 100644 --- a/test/core/end2end/tests/simple_delayed_request.cc +++ b/test/core/end2end/tests/simple_delayed_request.cc @@ -169,7 +169,7 @@ static void simple_delayed_request_body(grpc_end2end_test_config config, GPR_ASSERT(status == GRPC_STATUS_UNIMPLEMENTED); GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz")); GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/foo")); - GPR_ASSERT(was_cancelled == 1); + GPR_ASSERT(was_cancelled == 0); grpc_slice_unref(details); grpc_metadata_array_destroy(&initial_metadata_recv); diff --git a/test/core/end2end/tests/simple_request.cc b/test/core/end2end/tests/simple_request.cc index 8e0f863d847..afeb43df057 100644 --- a/test/core/end2end/tests/simple_request.cc +++ b/test/core/end2end/tests/simple_request.cc @@ -225,7 +225,7 @@ static void simple_request_body(grpc_end2end_test_config config, GPR_ASSERT(nullptr != strstr(error_string, "grpc_status")); GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/foo")); GPR_ASSERT(0 == call_details.flags); - GPR_ASSERT(was_cancelled == 1); + GPR_ASSERT(was_cancelled == 0); grpc_slice_unref(details); gpr_free((void*)error_string); diff --git a/test/core/end2end/tests/streaming_error_response.cc b/test/core/end2end/tests/streaming_error_response.cc index ac783611a7b..8d195be6abf 100644 --- a/test/core/end2end/tests/streaming_error_response.cc +++ b/test/core/end2end/tests/streaming_error_response.cc @@ -264,7 +264,7 @@ static void test(grpc_end2end_test_config config, bool request_status_early, GPR_ASSERT(status == GRPC_STATUS_FAILED_PRECONDITION); GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz")); GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/foo")); - GPR_ASSERT(was_cancelled == 1); + GPR_ASSERT(was_cancelled == 0); grpc_slice_unref(details); grpc_metadata_array_destroy(&initial_metadata_recv); diff --git a/test/cpp/end2end/async_end2end_test.cc b/test/cpp/end2end/async_end2end_test.cc index 5c3ced48575..1f1b2db855a 100644 --- a/test/cpp/end2end/async_end2end_test.cc +++ b/test/cpp/end2end/async_end2end_test.cc @@ -351,6 +351,52 @@ TEST_P(AsyncEnd2endTest, SimpleRpc) { SendRpc(1); } +TEST_P(AsyncEnd2endTest, SimpleRpcWithExpectedError) { + ResetStub(); + + EchoRequest send_request; + EchoRequest recv_request; + EchoResponse send_response; + EchoResponse recv_response; + Status recv_status; + + ClientContext cli_ctx; + ServerContext srv_ctx; + grpc::ServerAsyncResponseWriter response_writer(&srv_ctx); + ErrorStatus error_status; + + send_request.set_message(GetParam().message_content); + error_status.set_code(1); // CANCELLED + error_status.set_error_message("cancel error message"); + *send_request.mutable_param()->mutable_expected_error() = error_status; + + std::unique_ptr> response_reader( + stub_->AsyncEcho(&cli_ctx, send_request, cq_.get())); + + srv_ctx.AsyncNotifyWhenDone(tag(5)); + service_->RequestEcho(&srv_ctx, &recv_request, &response_writer, cq_.get(), + cq_.get(), tag(2)); + + response_reader->Finish(&recv_response, &recv_status, tag(4)); + + Verifier().Expect(2, true).Verify(cq_.get()); + EXPECT_EQ(send_request.message(), recv_request.message()); + + send_response.set_message(recv_request.message()); + response_writer.Finish( + send_response, + Status( + static_cast(recv_request.param().expected_error().code()), + recv_request.param().expected_error().error_message()), + tag(3)); + Verifier().Expect(3, true).Expect(4, true).Expect(5, true).Verify(cq_.get()); + + EXPECT_EQ(recv_response.message(), ""); + EXPECT_EQ(recv_status.error_code(), error_status.code()); + EXPECT_EQ(recv_status.error_message(), error_status.error_message()); + EXPECT_FALSE(srv_ctx.IsCancelled()); +} + TEST_P(AsyncEnd2endTest, SequentialRpcs) { ResetStub(); SendRpc(10); @@ -1856,9 +1902,10 @@ std::vector CreateTestScenarios(bool /*test_secure*/, } #ifndef MEMORY_SANITIZER // 4MB message processing with SSL is very slow under msan - // (causes timeouts) and doesn't really increase the signal from tests + // (causes timeouts) and doesn't really increase the signal from tests. + // Reserve 100 bytes for other fields of the message proto. messages.push_back( - grpc::string(GRPC_DEFAULT_MAX_RECV_MESSAGE_LENGTH - 10, 'a')); + grpc::string(GRPC_DEFAULT_MAX_RECV_MESSAGE_LENGTH - 100, 'a')); #endif } diff --git a/test/cpp/end2end/client_callback_end2end_test.cc b/test/cpp/end2end/client_callback_end2end_test.cc index 4f8bfeba372..05aac336789 100644 --- a/test/cpp/end2end/client_callback_end2end_test.cc +++ b/test/cpp/end2end/client_callback_end2end_test.cc @@ -385,6 +385,41 @@ TEST_P(ClientCallbackEnd2endTest, SimpleRpc) { SendRpcs(1, false); } +TEST_P(ClientCallbackEnd2endTest, SimpleRpcExpectedError) { + MAYBE_SKIP_TEST; + ResetStub(); + + EchoRequest request; + EchoResponse response; + ClientContext cli_ctx; + ErrorStatus error_status; + + request.set_message("Hello failure"); + error_status.set_code(1); // CANCELLED + error_status.set_error_message("cancel error message"); + *request.mutable_param()->mutable_expected_error() = error_status; + + std::mutex mu; + std::condition_variable cv; + bool done = false; + + stub_->experimental_async()->Echo( + &cli_ctx, &request, &response, + [&response, &done, &mu, &cv, &error_status](Status s) { + EXPECT_EQ("", response.message()); + EXPECT_EQ(error_status.code(), s.error_code()); + EXPECT_EQ(error_status.error_message(), s.error_message()); + std::lock_guard l(mu); + done = true; + cv.notify_one(); + }); + + std::unique_lock l(mu); + while (!done) { + cv.wait(l); + } +} + TEST_P(ClientCallbackEnd2endTest, SimpleRpcUnderLockNested) { MAYBE_SKIP_TEST; ResetStub(); diff --git a/test/cpp/end2end/test_service_impl.cc b/test/cpp/end2end/test_service_impl.cc index 285f8ae4f98..c3391bb1243 100644 --- a/test/cpp/end2end/test_service_impl.cc +++ b/test/cpp/end2end/test_service_impl.cc @@ -185,6 +185,10 @@ experimental::ServerUnaryReactor* CallbackTestServiceImpl::Echo( EXPECT_TRUE(initial_metadata_sent_); } EXPECT_EQ(ctx_->IsCancelled(), on_cancel_invoked_); + // Validate that finishing with a non-OK status doesn't cause cancellation + if (req_->has_param() && req_->param().has_expected_error()) { + EXPECT_FALSE(on_cancel_invoked_); + } async_cancel_check_.join(); if (rpc_wait_thread_.joinable()) { rpc_wait_thread_.join(); diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index f46683470da..5242b54e13a 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC Core" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 10.0.0 +PROJECT_NUMBER = 11.0.0 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index b026c8b1f8a..b0a051f81b8 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC Core" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 10.0.0 +PROJECT_NUMBER = 11.0.0 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a From fa1c66852b35ef39be61fb03748bfb2c2a1bd311 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Thu, 11 Jun 2020 14:43:13 -0700 Subject: [PATCH 26/80] Increase reconnect sleep time to 5x poller interval --- test/cpp/end2end/end2end_test.cc | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/test/cpp/end2end/end2end_test.cc b/test/cpp/end2end/end2end_test.cc index 6690d320043..79b5f4bbcc1 100644 --- a/test/cpp/end2end/end2end_test.cc +++ b/test/cpp/end2end/end2end_test.cc @@ -86,6 +86,8 @@ bool CheckIsLocalhost(const grpc::string& addr) { addr.substr(0, kIpv6.size()) == kIpv6; } +const int kClientChannelBackupPollIntervalMs = 200; + const char kTestCredsPluginErrorMsg[] = "Could not find plugin metadata."; const char kFakeToken[] = "fake_token"; @@ -916,12 +918,13 @@ TEST_P(End2endTest, ReconnectChannel) { SendRpc(stub_.get(), 1, false); RestartServer(std::shared_ptr()); // It needs more than GRPC_CLIENT_CHANNEL_BACKUP_POLL_INTERVAL_MS time to - // reconnect the channel. - gpr_sleep_until(gpr_time_add( - gpr_now(GPR_CLOCK_REALTIME), - gpr_time_from_millis( - 300 * poller_slowdown_factor * grpc_test_slowdown_factor(), - GPR_TIMESPAN))); + // reconnect the channel. Make it a factor of 5x + gpr_sleep_until( + gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), + gpr_time_from_millis(kClientChannelBackupPollIntervalMs * 5 * + poller_slowdown_factor * + grpc_test_slowdown_factor(), + GPR_TIMESPAN))); SendRpc(stub_.get(), 1, false); } @@ -2273,7 +2276,8 @@ std::vector CreateTestScenarios(bool use_proxy, std::vector scenarios; std::vector credentials_types; - GPR_GLOBAL_CONFIG_SET(grpc_client_channel_backup_poll_interval_ms, 200); + GPR_GLOBAL_CONFIG_SET(grpc_client_channel_backup_poll_interval_ms, + kClientChannelBackupPollIntervalMs); #if TARGET_OS_IPHONE // Workaround Apple CFStream bug gpr_setenv("grpc_cfstream", "0"); From 308fbc6ade005c5aa95cf9e4165ac3ca1ef3c874 Mon Sep 17 00:00:00 2001 From: michaelywg Date: Thu, 11 Jun 2020 15:47:23 -0700 Subject: [PATCH 27/80] Added rbac.proto and dependencies, then ran gen_upb_api.sh --- .../envoy/config/rbac/v2/rbac.upb.c | 173 ++++ .../envoy/config/rbac/v2/rbac.upb.h | 615 ++++++++++++++ .../envoy/type/matcher/path.upb.c | 33 + .../envoy/type/matcher/path.upb.h | 72 ++ .../google/api/expr/v1alpha1/syntax.upb.c | 234 ++++++ .../google/api/expr/v1alpha1/syntax.upb.h | 760 ++++++++++++++++++ tools/codegen/core/gen_upb_api.sh | 3 + 7 files changed, 1890 insertions(+) create mode 100644 src/core/ext/upb-generated/envoy/config/rbac/v2/rbac.upb.c create mode 100644 src/core/ext/upb-generated/envoy/config/rbac/v2/rbac.upb.h create mode 100644 src/core/ext/upb-generated/envoy/type/matcher/path.upb.c create mode 100644 src/core/ext/upb-generated/envoy/type/matcher/path.upb.h create mode 100644 src/core/ext/upb-generated/google/api/expr/v1alpha1/syntax.upb.c create mode 100644 src/core/ext/upb-generated/google/api/expr/v1alpha1/syntax.upb.h diff --git a/src/core/ext/upb-generated/envoy/config/rbac/v2/rbac.upb.c b/src/core/ext/upb-generated/envoy/config/rbac/v2/rbac.upb.c new file mode 100644 index 00000000000..55c543c01d6 --- /dev/null +++ b/src/core/ext/upb-generated/envoy/config/rbac/v2/rbac.upb.c @@ -0,0 +1,173 @@ +/* This file was generated by upbc (the upb compiler) from the input + * file: + * + * envoy/config/rbac/v2/rbac.proto + * + * Do not edit -- your changes will be discarded when the file is + * regenerated. */ + +#include +#include "upb/msg.h" +#include "envoy/config/rbac/v2/rbac.upb.h" +#include "envoy/api/v2/core/address.upb.h" +#include "envoy/api/v2/route/route_components.upb.h" +#include "envoy/type/matcher/metadata.upb.h" +#include "envoy/type/matcher/path.upb.h" +#include "envoy/type/matcher/string.upb.h" +#include "google/api/expr/v1alpha1/syntax.upb.h" +#include "udpa/annotations/status.upb.h" +#include "validate/validate.upb.h" + +#include "upb/port_def.inc" + +static const upb_msglayout *const envoy_config_rbac_v2_RBAC_submsgs[1] = { + &envoy_config_rbac_v2_RBAC_PoliciesEntry_msginit, +}; + +static const upb_msglayout_field envoy_config_rbac_v2_RBAC__fields[2] = { + {1, UPB_SIZE(0, 0), 0, 0, 14, 1}, + {2, UPB_SIZE(8, 8), 0, 0, 11, 3}, +}; + +const upb_msglayout envoy_config_rbac_v2_RBAC_msginit = { + &envoy_config_rbac_v2_RBAC_submsgs[0], + &envoy_config_rbac_v2_RBAC__fields[0], + UPB_SIZE(16, 16), 2, false, +}; + +static const upb_msglayout *const envoy_config_rbac_v2_RBAC_PoliciesEntry_submsgs[1] = { + &envoy_config_rbac_v2_Policy_msginit, +}; + +static const upb_msglayout_field envoy_config_rbac_v2_RBAC_PoliciesEntry__fields[2] = { + {1, UPB_SIZE(0, 0), 0, 0, 9, 1}, + {2, UPB_SIZE(8, 16), 0, 0, 11, 1}, +}; + +const upb_msglayout envoy_config_rbac_v2_RBAC_PoliciesEntry_msginit = { + &envoy_config_rbac_v2_RBAC_PoliciesEntry_submsgs[0], + &envoy_config_rbac_v2_RBAC_PoliciesEntry__fields[0], + UPB_SIZE(16, 32), 2, false, +}; + +static const upb_msglayout *const envoy_config_rbac_v2_Policy_submsgs[3] = { + &envoy_config_rbac_v2_Permission_msginit, + &envoy_config_rbac_v2_Principal_msginit, + &google_api_expr_v1alpha1_Expr_msginit, +}; + +static const upb_msglayout_field envoy_config_rbac_v2_Policy__fields[3] = { + {1, UPB_SIZE(4, 8), 0, 0, 11, 3}, + {2, UPB_SIZE(8, 16), 0, 1, 11, 3}, + {3, UPB_SIZE(0, 0), 0, 2, 11, 1}, +}; + +const upb_msglayout envoy_config_rbac_v2_Policy_msginit = { + &envoy_config_rbac_v2_Policy_submsgs[0], + &envoy_config_rbac_v2_Policy__fields[0], + UPB_SIZE(12, 24), 3, false, +}; + +static const upb_msglayout *const envoy_config_rbac_v2_Permission_submsgs[8] = { + &envoy_api_v2_core_CidrRange_msginit, + &envoy_api_v2_route_HeaderMatcher_msginit, + &envoy_config_rbac_v2_Permission_msginit, + &envoy_config_rbac_v2_Permission_Set_msginit, + &envoy_type_matcher_MetadataMatcher_msginit, + &envoy_type_matcher_PathMatcher_msginit, + &envoy_type_matcher_StringMatcher_msginit, +}; + +static const upb_msglayout_field envoy_config_rbac_v2_Permission__fields[10] = { + {1, UPB_SIZE(0, 0), UPB_SIZE(-5, -9), 3, 11, 1}, + {2, UPB_SIZE(0, 0), UPB_SIZE(-5, -9), 3, 11, 1}, + {3, UPB_SIZE(0, 0), UPB_SIZE(-5, -9), 0, 8, 1}, + {4, UPB_SIZE(0, 0), UPB_SIZE(-5, -9), 1, 11, 1}, + {5, UPB_SIZE(0, 0), UPB_SIZE(-5, -9), 0, 11, 1}, + {6, UPB_SIZE(0, 0), UPB_SIZE(-5, -9), 0, 13, 1}, + {7, UPB_SIZE(0, 0), UPB_SIZE(-5, -9), 4, 11, 1}, + {8, UPB_SIZE(0, 0), UPB_SIZE(-5, -9), 2, 11, 1}, + {9, UPB_SIZE(0, 0), UPB_SIZE(-5, -9), 6, 11, 1}, + {10, UPB_SIZE(0, 0), UPB_SIZE(-5, -9), 5, 11, 1}, +}; + +const upb_msglayout envoy_config_rbac_v2_Permission_msginit = { + &envoy_config_rbac_v2_Permission_submsgs[0], + &envoy_config_rbac_v2_Permission__fields[0], + UPB_SIZE(8, 16), 10, false, +}; + +static const upb_msglayout *const envoy_config_rbac_v2_Permission_Set_submsgs[1] = { + &envoy_config_rbac_v2_Permission_msginit, +}; + +static const upb_msglayout_field envoy_config_rbac_v2_Permission_Set__fields[1] = { + {1, UPB_SIZE(0, 0), 0, 0, 11, 3}, +}; + +const upb_msglayout envoy_config_rbac_v2_Permission_Set_msginit = { + &envoy_config_rbac_v2_Permission_Set_submsgs[0], + &envoy_config_rbac_v2_Permission_Set__fields[0], + UPB_SIZE(4, 8), 1, false, +}; + +static const upb_msglayout *const envoy_config_rbac_v2_Principal_submsgs[10] = { + &envoy_api_v2_core_CidrRange_msginit, + &envoy_api_v2_route_HeaderMatcher_msginit, + &envoy_config_rbac_v2_Principal_msginit, + &envoy_config_rbac_v2_Principal_Authenticated_msginit, + &envoy_config_rbac_v2_Principal_Set_msginit, + &envoy_type_matcher_MetadataMatcher_msginit, + &envoy_type_matcher_PathMatcher_msginit, +}; + +static const upb_msglayout_field envoy_config_rbac_v2_Principal__fields[11] = { + {1, UPB_SIZE(0, 0), UPB_SIZE(-5, -9), 4, 11, 1}, + {2, UPB_SIZE(0, 0), UPB_SIZE(-5, -9), 4, 11, 1}, + {3, UPB_SIZE(0, 0), UPB_SIZE(-5, -9), 0, 8, 1}, + {4, UPB_SIZE(0, 0), UPB_SIZE(-5, -9), 3, 11, 1}, + {5, UPB_SIZE(0, 0), UPB_SIZE(-5, -9), 0, 11, 1}, + {6, UPB_SIZE(0, 0), UPB_SIZE(-5, -9), 1, 11, 1}, + {7, UPB_SIZE(0, 0), UPB_SIZE(-5, -9), 5, 11, 1}, + {8, UPB_SIZE(0, 0), UPB_SIZE(-5, -9), 2, 11, 1}, + {9, UPB_SIZE(0, 0), UPB_SIZE(-5, -9), 6, 11, 1}, + {10, UPB_SIZE(0, 0), UPB_SIZE(-5, -9), 0, 11, 1}, + {11, UPB_SIZE(0, 0), UPB_SIZE(-5, -9), 0, 11, 1}, +}; + +const upb_msglayout envoy_config_rbac_v2_Principal_msginit = { + &envoy_config_rbac_v2_Principal_submsgs[0], + &envoy_config_rbac_v2_Principal__fields[0], + UPB_SIZE(8, 16), 11, false, +}; + +static const upb_msglayout *const envoy_config_rbac_v2_Principal_Set_submsgs[1] = { + &envoy_config_rbac_v2_Principal_msginit, +}; + +static const upb_msglayout_field envoy_config_rbac_v2_Principal_Set__fields[1] = { + {1, UPB_SIZE(0, 0), 0, 0, 11, 3}, +}; + +const upb_msglayout envoy_config_rbac_v2_Principal_Set_msginit = { + &envoy_config_rbac_v2_Principal_Set_submsgs[0], + &envoy_config_rbac_v2_Principal_Set__fields[0], + UPB_SIZE(4, 8), 1, false, +}; + +static const upb_msglayout *const envoy_config_rbac_v2_Principal_Authenticated_submsgs[1] = { + &envoy_type_matcher_StringMatcher_msginit, +}; + +static const upb_msglayout_field envoy_config_rbac_v2_Principal_Authenticated__fields[1] = { + {2, UPB_SIZE(0, 0), 0, 0, 11, 1}, +}; + +const upb_msglayout envoy_config_rbac_v2_Principal_Authenticated_msginit = { + &envoy_config_rbac_v2_Principal_Authenticated_submsgs[0], + &envoy_config_rbac_v2_Principal_Authenticated__fields[0], + UPB_SIZE(4, 8), 1, false, +}; + +#include "upb/port_undef.inc" + diff --git a/src/core/ext/upb-generated/envoy/config/rbac/v2/rbac.upb.h b/src/core/ext/upb-generated/envoy/config/rbac/v2/rbac.upb.h new file mode 100644 index 00000000000..0330bdf113b --- /dev/null +++ b/src/core/ext/upb-generated/envoy/config/rbac/v2/rbac.upb.h @@ -0,0 +1,615 @@ +/* This file was generated by upbc (the upb compiler) from the input + * file: + * + * envoy/config/rbac/v2/rbac.proto + * + * Do not edit -- your changes will be discarded when the file is + * regenerated. */ + +#ifndef ENVOY_CONFIG_RBAC_V2_RBAC_PROTO_UPB_H_ +#define ENVOY_CONFIG_RBAC_V2_RBAC_PROTO_UPB_H_ + +#include "upb/generated_util.h" +#include "upb/msg.h" +#include "upb/decode.h" +#include "upb/encode.h" + +#include "upb/port_def.inc" + +#ifdef __cplusplus +extern "C" { +#endif + +struct envoy_config_rbac_v2_RBAC; +struct envoy_config_rbac_v2_RBAC_PoliciesEntry; +struct envoy_config_rbac_v2_Policy; +struct envoy_config_rbac_v2_Permission; +struct envoy_config_rbac_v2_Permission_Set; +struct envoy_config_rbac_v2_Principal; +struct envoy_config_rbac_v2_Principal_Set; +struct envoy_config_rbac_v2_Principal_Authenticated; +typedef struct envoy_config_rbac_v2_RBAC envoy_config_rbac_v2_RBAC; +typedef struct envoy_config_rbac_v2_RBAC_PoliciesEntry envoy_config_rbac_v2_RBAC_PoliciesEntry; +typedef struct envoy_config_rbac_v2_Policy envoy_config_rbac_v2_Policy; +typedef struct envoy_config_rbac_v2_Permission envoy_config_rbac_v2_Permission; +typedef struct envoy_config_rbac_v2_Permission_Set envoy_config_rbac_v2_Permission_Set; +typedef struct envoy_config_rbac_v2_Principal envoy_config_rbac_v2_Principal; +typedef struct envoy_config_rbac_v2_Principal_Set envoy_config_rbac_v2_Principal_Set; +typedef struct envoy_config_rbac_v2_Principal_Authenticated envoy_config_rbac_v2_Principal_Authenticated; +extern const upb_msglayout envoy_config_rbac_v2_RBAC_msginit; +extern const upb_msglayout envoy_config_rbac_v2_RBAC_PoliciesEntry_msginit; +extern const upb_msglayout envoy_config_rbac_v2_Policy_msginit; +extern const upb_msglayout envoy_config_rbac_v2_Permission_msginit; +extern const upb_msglayout envoy_config_rbac_v2_Permission_Set_msginit; +extern const upb_msglayout envoy_config_rbac_v2_Principal_msginit; +extern const upb_msglayout envoy_config_rbac_v2_Principal_Set_msginit; +extern const upb_msglayout envoy_config_rbac_v2_Principal_Authenticated_msginit; +struct envoy_api_v2_core_CidrRange; +struct envoy_api_v2_route_HeaderMatcher; +struct envoy_type_matcher_MetadataMatcher; +struct envoy_type_matcher_PathMatcher; +struct envoy_type_matcher_StringMatcher; +struct google_api_expr_v1alpha1_Expr; +extern const upb_msglayout envoy_api_v2_core_CidrRange_msginit; +extern const upb_msglayout envoy_api_v2_route_HeaderMatcher_msginit; +extern const upb_msglayout envoy_type_matcher_MetadataMatcher_msginit; +extern const upb_msglayout envoy_type_matcher_PathMatcher_msginit; +extern const upb_msglayout envoy_type_matcher_StringMatcher_msginit; +extern const upb_msglayout google_api_expr_v1alpha1_Expr_msginit; + +typedef enum { + envoy_config_rbac_v2_RBAC_ALLOW = 0, + envoy_config_rbac_v2_RBAC_DENY = 1 +} envoy_config_rbac_v2_RBAC_Action; + + +/* envoy.config.rbac.v2.RBAC */ + +UPB_INLINE envoy_config_rbac_v2_RBAC *envoy_config_rbac_v2_RBAC_new(upb_arena *arena) { + return (envoy_config_rbac_v2_RBAC *)upb_msg_new(&envoy_config_rbac_v2_RBAC_msginit, arena); +} +UPB_INLINE envoy_config_rbac_v2_RBAC *envoy_config_rbac_v2_RBAC_parse(const char *buf, size_t size, + upb_arena *arena) { + envoy_config_rbac_v2_RBAC *ret = envoy_config_rbac_v2_RBAC_new(arena); + return (ret && upb_decode(buf, size, ret, &envoy_config_rbac_v2_RBAC_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *envoy_config_rbac_v2_RBAC_serialize(const envoy_config_rbac_v2_RBAC *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &envoy_config_rbac_v2_RBAC_msginit, arena, len); +} + +UPB_INLINE int32_t envoy_config_rbac_v2_RBAC_action(const envoy_config_rbac_v2_RBAC *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(0, 0)); } +UPB_INLINE const envoy_config_rbac_v2_RBAC_PoliciesEntry* const* envoy_config_rbac_v2_RBAC_policies(const envoy_config_rbac_v2_RBAC *msg, size_t *len) { return (const envoy_config_rbac_v2_RBAC_PoliciesEntry* const*)_upb_array_accessor(msg, UPB_SIZE(8, 8), len); } + +UPB_INLINE void envoy_config_rbac_v2_RBAC_set_action(envoy_config_rbac_v2_RBAC *msg, int32_t value) { + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(0, 0)) = value; +} +UPB_INLINE envoy_config_rbac_v2_RBAC_PoliciesEntry** envoy_config_rbac_v2_RBAC_mutable_policies(envoy_config_rbac_v2_RBAC *msg, size_t *len) { + return (envoy_config_rbac_v2_RBAC_PoliciesEntry**)_upb_array_mutable_accessor(msg, UPB_SIZE(8, 8), len); +} +UPB_INLINE envoy_config_rbac_v2_RBAC_PoliciesEntry** envoy_config_rbac_v2_RBAC_resize_policies(envoy_config_rbac_v2_RBAC *msg, size_t len, upb_arena *arena) { + return (envoy_config_rbac_v2_RBAC_PoliciesEntry**)_upb_array_resize_accessor(msg, UPB_SIZE(8, 8), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct envoy_config_rbac_v2_RBAC_PoliciesEntry* envoy_config_rbac_v2_RBAC_add_policies(envoy_config_rbac_v2_RBAC *msg, upb_arena *arena) { + struct envoy_config_rbac_v2_RBAC_PoliciesEntry* sub = (struct envoy_config_rbac_v2_RBAC_PoliciesEntry*)upb_msg_new(&envoy_config_rbac_v2_RBAC_PoliciesEntry_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(8, 8), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} + +/* envoy.config.rbac.v2.RBAC.PoliciesEntry */ + +UPB_INLINE envoy_config_rbac_v2_RBAC_PoliciesEntry *envoy_config_rbac_v2_RBAC_PoliciesEntry_new(upb_arena *arena) { + return (envoy_config_rbac_v2_RBAC_PoliciesEntry *)upb_msg_new(&envoy_config_rbac_v2_RBAC_PoliciesEntry_msginit, arena); +} +UPB_INLINE envoy_config_rbac_v2_RBAC_PoliciesEntry *envoy_config_rbac_v2_RBAC_PoliciesEntry_parse(const char *buf, size_t size, + upb_arena *arena) { + envoy_config_rbac_v2_RBAC_PoliciesEntry *ret = envoy_config_rbac_v2_RBAC_PoliciesEntry_new(arena); + return (ret && upb_decode(buf, size, ret, &envoy_config_rbac_v2_RBAC_PoliciesEntry_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *envoy_config_rbac_v2_RBAC_PoliciesEntry_serialize(const envoy_config_rbac_v2_RBAC_PoliciesEntry *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &envoy_config_rbac_v2_RBAC_PoliciesEntry_msginit, arena, len); +} + +UPB_INLINE upb_strview envoy_config_rbac_v2_RBAC_PoliciesEntry_key(const envoy_config_rbac_v2_RBAC_PoliciesEntry *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(0, 0)); } +UPB_INLINE const envoy_config_rbac_v2_Policy* envoy_config_rbac_v2_RBAC_PoliciesEntry_value(const envoy_config_rbac_v2_RBAC_PoliciesEntry *msg) { return UPB_FIELD_AT(msg, const envoy_config_rbac_v2_Policy*, UPB_SIZE(8, 16)); } + +UPB_INLINE void envoy_config_rbac_v2_RBAC_PoliciesEntry_set_key(envoy_config_rbac_v2_RBAC_PoliciesEntry *msg, upb_strview value) { + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(0, 0)) = value; +} +UPB_INLINE void envoy_config_rbac_v2_RBAC_PoliciesEntry_set_value(envoy_config_rbac_v2_RBAC_PoliciesEntry *msg, envoy_config_rbac_v2_Policy* value) { + UPB_FIELD_AT(msg, envoy_config_rbac_v2_Policy*, UPB_SIZE(8, 16)) = value; +} +UPB_INLINE struct envoy_config_rbac_v2_Policy* envoy_config_rbac_v2_RBAC_PoliciesEntry_mutable_value(envoy_config_rbac_v2_RBAC_PoliciesEntry *msg, upb_arena *arena) { + struct envoy_config_rbac_v2_Policy* sub = (struct envoy_config_rbac_v2_Policy*)envoy_config_rbac_v2_RBAC_PoliciesEntry_value(msg); + if (sub == NULL) { + sub = (struct envoy_config_rbac_v2_Policy*)upb_msg_new(&envoy_config_rbac_v2_Policy_msginit, arena); + if (!sub) return NULL; + envoy_config_rbac_v2_RBAC_PoliciesEntry_set_value(msg, sub); + } + return sub; +} + +/* envoy.config.rbac.v2.Policy */ + +UPB_INLINE envoy_config_rbac_v2_Policy *envoy_config_rbac_v2_Policy_new(upb_arena *arena) { + return (envoy_config_rbac_v2_Policy *)upb_msg_new(&envoy_config_rbac_v2_Policy_msginit, arena); +} +UPB_INLINE envoy_config_rbac_v2_Policy *envoy_config_rbac_v2_Policy_parse(const char *buf, size_t size, + upb_arena *arena) { + envoy_config_rbac_v2_Policy *ret = envoy_config_rbac_v2_Policy_new(arena); + return (ret && upb_decode(buf, size, ret, &envoy_config_rbac_v2_Policy_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *envoy_config_rbac_v2_Policy_serialize(const envoy_config_rbac_v2_Policy *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &envoy_config_rbac_v2_Policy_msginit, arena, len); +} + +UPB_INLINE const envoy_config_rbac_v2_Permission* const* envoy_config_rbac_v2_Policy_permissions(const envoy_config_rbac_v2_Policy *msg, size_t *len) { return (const envoy_config_rbac_v2_Permission* const*)_upb_array_accessor(msg, UPB_SIZE(4, 8), len); } +UPB_INLINE const envoy_config_rbac_v2_Principal* const* envoy_config_rbac_v2_Policy_principals(const envoy_config_rbac_v2_Policy *msg, size_t *len) { return (const envoy_config_rbac_v2_Principal* const*)_upb_array_accessor(msg, UPB_SIZE(8, 16), len); } +UPB_INLINE const struct google_api_expr_v1alpha1_Expr* envoy_config_rbac_v2_Policy_condition(const envoy_config_rbac_v2_Policy *msg) { return UPB_FIELD_AT(msg, const struct google_api_expr_v1alpha1_Expr*, UPB_SIZE(0, 0)); } + +UPB_INLINE envoy_config_rbac_v2_Permission** envoy_config_rbac_v2_Policy_mutable_permissions(envoy_config_rbac_v2_Policy *msg, size_t *len) { + return (envoy_config_rbac_v2_Permission**)_upb_array_mutable_accessor(msg, UPB_SIZE(4, 8), len); +} +UPB_INLINE envoy_config_rbac_v2_Permission** envoy_config_rbac_v2_Policy_resize_permissions(envoy_config_rbac_v2_Policy *msg, size_t len, upb_arena *arena) { + return (envoy_config_rbac_v2_Permission**)_upb_array_resize_accessor(msg, UPB_SIZE(4, 8), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct envoy_config_rbac_v2_Permission* envoy_config_rbac_v2_Policy_add_permissions(envoy_config_rbac_v2_Policy *msg, upb_arena *arena) { + struct envoy_config_rbac_v2_Permission* sub = (struct envoy_config_rbac_v2_Permission*)upb_msg_new(&envoy_config_rbac_v2_Permission_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(4, 8), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} +UPB_INLINE envoy_config_rbac_v2_Principal** envoy_config_rbac_v2_Policy_mutable_principals(envoy_config_rbac_v2_Policy *msg, size_t *len) { + return (envoy_config_rbac_v2_Principal**)_upb_array_mutable_accessor(msg, UPB_SIZE(8, 16), len); +} +UPB_INLINE envoy_config_rbac_v2_Principal** envoy_config_rbac_v2_Policy_resize_principals(envoy_config_rbac_v2_Policy *msg, size_t len, upb_arena *arena) { + return (envoy_config_rbac_v2_Principal**)_upb_array_resize_accessor(msg, UPB_SIZE(8, 16), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct envoy_config_rbac_v2_Principal* envoy_config_rbac_v2_Policy_add_principals(envoy_config_rbac_v2_Policy *msg, upb_arena *arena) { + struct envoy_config_rbac_v2_Principal* sub = (struct envoy_config_rbac_v2_Principal*)upb_msg_new(&envoy_config_rbac_v2_Principal_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(8, 16), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} +UPB_INLINE void envoy_config_rbac_v2_Policy_set_condition(envoy_config_rbac_v2_Policy *msg, struct google_api_expr_v1alpha1_Expr* value) { + UPB_FIELD_AT(msg, struct google_api_expr_v1alpha1_Expr*, UPB_SIZE(0, 0)) = value; +} +UPB_INLINE struct google_api_expr_v1alpha1_Expr* envoy_config_rbac_v2_Policy_mutable_condition(envoy_config_rbac_v2_Policy *msg, upb_arena *arena) { + struct google_api_expr_v1alpha1_Expr* sub = (struct google_api_expr_v1alpha1_Expr*)envoy_config_rbac_v2_Policy_condition(msg); + if (sub == NULL) { + sub = (struct google_api_expr_v1alpha1_Expr*)upb_msg_new(&google_api_expr_v1alpha1_Expr_msginit, arena); + if (!sub) return NULL; + envoy_config_rbac_v2_Policy_set_condition(msg, sub); + } + return sub; +} + +/* envoy.config.rbac.v2.Permission */ + +UPB_INLINE envoy_config_rbac_v2_Permission *envoy_config_rbac_v2_Permission_new(upb_arena *arena) { + return (envoy_config_rbac_v2_Permission *)upb_msg_new(&envoy_config_rbac_v2_Permission_msginit, arena); +} +UPB_INLINE envoy_config_rbac_v2_Permission *envoy_config_rbac_v2_Permission_parse(const char *buf, size_t size, + upb_arena *arena) { + envoy_config_rbac_v2_Permission *ret = envoy_config_rbac_v2_Permission_new(arena); + return (ret && upb_decode(buf, size, ret, &envoy_config_rbac_v2_Permission_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *envoy_config_rbac_v2_Permission_serialize(const envoy_config_rbac_v2_Permission *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &envoy_config_rbac_v2_Permission_msginit, arena, len); +} + +typedef enum { + envoy_config_rbac_v2_Permission_rule_and_rules = 1, + envoy_config_rbac_v2_Permission_rule_or_rules = 2, + envoy_config_rbac_v2_Permission_rule_any = 3, + envoy_config_rbac_v2_Permission_rule_header = 4, + envoy_config_rbac_v2_Permission_rule_url_path = 10, + envoy_config_rbac_v2_Permission_rule_destination_ip = 5, + envoy_config_rbac_v2_Permission_rule_destination_port = 6, + envoy_config_rbac_v2_Permission_rule_metadata = 7, + envoy_config_rbac_v2_Permission_rule_not_rule = 8, + envoy_config_rbac_v2_Permission_rule_requested_server_name = 9, + envoy_config_rbac_v2_Permission_rule_NOT_SET = 0 +} envoy_config_rbac_v2_Permission_rule_oneofcases; +UPB_INLINE envoy_config_rbac_v2_Permission_rule_oneofcases envoy_config_rbac_v2_Permission_rule_case(const envoy_config_rbac_v2_Permission* msg) { return (envoy_config_rbac_v2_Permission_rule_oneofcases)UPB_FIELD_AT(msg, int32_t, UPB_SIZE(4, 8)); } + +UPB_INLINE bool envoy_config_rbac_v2_Permission_has_and_rules(const envoy_config_rbac_v2_Permission *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(4, 8), 1); } +UPB_INLINE const envoy_config_rbac_v2_Permission_Set* envoy_config_rbac_v2_Permission_and_rules(const envoy_config_rbac_v2_Permission *msg) { return UPB_READ_ONEOF(msg, const envoy_config_rbac_v2_Permission_Set*, UPB_SIZE(0, 0), UPB_SIZE(4, 8), 1, NULL); } +UPB_INLINE bool envoy_config_rbac_v2_Permission_has_or_rules(const envoy_config_rbac_v2_Permission *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(4, 8), 2); } +UPB_INLINE const envoy_config_rbac_v2_Permission_Set* envoy_config_rbac_v2_Permission_or_rules(const envoy_config_rbac_v2_Permission *msg) { return UPB_READ_ONEOF(msg, const envoy_config_rbac_v2_Permission_Set*, UPB_SIZE(0, 0), UPB_SIZE(4, 8), 2, NULL); } +UPB_INLINE bool envoy_config_rbac_v2_Permission_has_any(const envoy_config_rbac_v2_Permission *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(4, 8), 3); } +UPB_INLINE bool envoy_config_rbac_v2_Permission_any(const envoy_config_rbac_v2_Permission *msg) { return UPB_READ_ONEOF(msg, bool, UPB_SIZE(0, 0), UPB_SIZE(4, 8), 3, false); } +UPB_INLINE bool envoy_config_rbac_v2_Permission_has_header(const envoy_config_rbac_v2_Permission *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(4, 8), 4); } +UPB_INLINE const struct envoy_api_v2_route_HeaderMatcher* envoy_config_rbac_v2_Permission_header(const envoy_config_rbac_v2_Permission *msg) { return UPB_READ_ONEOF(msg, const struct envoy_api_v2_route_HeaderMatcher*, UPB_SIZE(0, 0), UPB_SIZE(4, 8), 4, NULL); } +UPB_INLINE bool envoy_config_rbac_v2_Permission_has_destination_ip(const envoy_config_rbac_v2_Permission *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(4, 8), 5); } +UPB_INLINE const struct envoy_api_v2_core_CidrRange* envoy_config_rbac_v2_Permission_destination_ip(const envoy_config_rbac_v2_Permission *msg) { return UPB_READ_ONEOF(msg, const struct envoy_api_v2_core_CidrRange*, UPB_SIZE(0, 0), UPB_SIZE(4, 8), 5, NULL); } +UPB_INLINE bool envoy_config_rbac_v2_Permission_has_destination_port(const envoy_config_rbac_v2_Permission *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(4, 8), 6); } +UPB_INLINE uint32_t envoy_config_rbac_v2_Permission_destination_port(const envoy_config_rbac_v2_Permission *msg) { return UPB_READ_ONEOF(msg, uint32_t, UPB_SIZE(0, 0), UPB_SIZE(4, 8), 6, 0); } +UPB_INLINE bool envoy_config_rbac_v2_Permission_has_metadata(const envoy_config_rbac_v2_Permission *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(4, 8), 7); } +UPB_INLINE const struct envoy_type_matcher_MetadataMatcher* envoy_config_rbac_v2_Permission_metadata(const envoy_config_rbac_v2_Permission *msg) { return UPB_READ_ONEOF(msg, const struct envoy_type_matcher_MetadataMatcher*, UPB_SIZE(0, 0), UPB_SIZE(4, 8), 7, NULL); } +UPB_INLINE bool envoy_config_rbac_v2_Permission_has_not_rule(const envoy_config_rbac_v2_Permission *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(4, 8), 8); } +UPB_INLINE const envoy_config_rbac_v2_Permission* envoy_config_rbac_v2_Permission_not_rule(const envoy_config_rbac_v2_Permission *msg) { return UPB_READ_ONEOF(msg, const envoy_config_rbac_v2_Permission*, UPB_SIZE(0, 0), UPB_SIZE(4, 8), 8, NULL); } +UPB_INLINE bool envoy_config_rbac_v2_Permission_has_requested_server_name(const envoy_config_rbac_v2_Permission *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(4, 8), 9); } +UPB_INLINE const struct envoy_type_matcher_StringMatcher* envoy_config_rbac_v2_Permission_requested_server_name(const envoy_config_rbac_v2_Permission *msg) { return UPB_READ_ONEOF(msg, const struct envoy_type_matcher_StringMatcher*, UPB_SIZE(0, 0), UPB_SIZE(4, 8), 9, NULL); } +UPB_INLINE bool envoy_config_rbac_v2_Permission_has_url_path(const envoy_config_rbac_v2_Permission *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(4, 8), 10); } +UPB_INLINE const struct envoy_type_matcher_PathMatcher* envoy_config_rbac_v2_Permission_url_path(const envoy_config_rbac_v2_Permission *msg) { return UPB_READ_ONEOF(msg, const struct envoy_type_matcher_PathMatcher*, UPB_SIZE(0, 0), UPB_SIZE(4, 8), 10, NULL); } + +UPB_INLINE void envoy_config_rbac_v2_Permission_set_and_rules(envoy_config_rbac_v2_Permission *msg, envoy_config_rbac_v2_Permission_Set* value) { + UPB_WRITE_ONEOF(msg, envoy_config_rbac_v2_Permission_Set*, UPB_SIZE(0, 0), value, UPB_SIZE(4, 8), 1); +} +UPB_INLINE struct envoy_config_rbac_v2_Permission_Set* envoy_config_rbac_v2_Permission_mutable_and_rules(envoy_config_rbac_v2_Permission *msg, upb_arena *arena) { + struct envoy_config_rbac_v2_Permission_Set* sub = (struct envoy_config_rbac_v2_Permission_Set*)envoy_config_rbac_v2_Permission_and_rules(msg); + if (sub == NULL) { + sub = (struct envoy_config_rbac_v2_Permission_Set*)upb_msg_new(&envoy_config_rbac_v2_Permission_Set_msginit, arena); + if (!sub) return NULL; + envoy_config_rbac_v2_Permission_set_and_rules(msg, sub); + } + return sub; +} +UPB_INLINE void envoy_config_rbac_v2_Permission_set_or_rules(envoy_config_rbac_v2_Permission *msg, envoy_config_rbac_v2_Permission_Set* value) { + UPB_WRITE_ONEOF(msg, envoy_config_rbac_v2_Permission_Set*, UPB_SIZE(0, 0), value, UPB_SIZE(4, 8), 2); +} +UPB_INLINE struct envoy_config_rbac_v2_Permission_Set* envoy_config_rbac_v2_Permission_mutable_or_rules(envoy_config_rbac_v2_Permission *msg, upb_arena *arena) { + struct envoy_config_rbac_v2_Permission_Set* sub = (struct envoy_config_rbac_v2_Permission_Set*)envoy_config_rbac_v2_Permission_or_rules(msg); + if (sub == NULL) { + sub = (struct envoy_config_rbac_v2_Permission_Set*)upb_msg_new(&envoy_config_rbac_v2_Permission_Set_msginit, arena); + if (!sub) return NULL; + envoy_config_rbac_v2_Permission_set_or_rules(msg, sub); + } + return sub; +} +UPB_INLINE void envoy_config_rbac_v2_Permission_set_any(envoy_config_rbac_v2_Permission *msg, bool value) { + UPB_WRITE_ONEOF(msg, bool, UPB_SIZE(0, 0), value, UPB_SIZE(4, 8), 3); +} +UPB_INLINE void envoy_config_rbac_v2_Permission_set_header(envoy_config_rbac_v2_Permission *msg, struct envoy_api_v2_route_HeaderMatcher* value) { + UPB_WRITE_ONEOF(msg, struct envoy_api_v2_route_HeaderMatcher*, UPB_SIZE(0, 0), value, UPB_SIZE(4, 8), 4); +} +UPB_INLINE struct envoy_api_v2_route_HeaderMatcher* envoy_config_rbac_v2_Permission_mutable_header(envoy_config_rbac_v2_Permission *msg, upb_arena *arena) { + struct envoy_api_v2_route_HeaderMatcher* sub = (struct envoy_api_v2_route_HeaderMatcher*)envoy_config_rbac_v2_Permission_header(msg); + if (sub == NULL) { + sub = (struct envoy_api_v2_route_HeaderMatcher*)upb_msg_new(&envoy_api_v2_route_HeaderMatcher_msginit, arena); + if (!sub) return NULL; + envoy_config_rbac_v2_Permission_set_header(msg, sub); + } + return sub; +} +UPB_INLINE void envoy_config_rbac_v2_Permission_set_destination_ip(envoy_config_rbac_v2_Permission *msg, struct envoy_api_v2_core_CidrRange* value) { + UPB_WRITE_ONEOF(msg, struct envoy_api_v2_core_CidrRange*, UPB_SIZE(0, 0), value, UPB_SIZE(4, 8), 5); +} +UPB_INLINE struct envoy_api_v2_core_CidrRange* envoy_config_rbac_v2_Permission_mutable_destination_ip(envoy_config_rbac_v2_Permission *msg, upb_arena *arena) { + struct envoy_api_v2_core_CidrRange* sub = (struct envoy_api_v2_core_CidrRange*)envoy_config_rbac_v2_Permission_destination_ip(msg); + if (sub == NULL) { + sub = (struct envoy_api_v2_core_CidrRange*)upb_msg_new(&envoy_api_v2_core_CidrRange_msginit, arena); + if (!sub) return NULL; + envoy_config_rbac_v2_Permission_set_destination_ip(msg, sub); + } + return sub; +} +UPB_INLINE void envoy_config_rbac_v2_Permission_set_destination_port(envoy_config_rbac_v2_Permission *msg, uint32_t value) { + UPB_WRITE_ONEOF(msg, uint32_t, UPB_SIZE(0, 0), value, UPB_SIZE(4, 8), 6); +} +UPB_INLINE void envoy_config_rbac_v2_Permission_set_metadata(envoy_config_rbac_v2_Permission *msg, struct envoy_type_matcher_MetadataMatcher* value) { + UPB_WRITE_ONEOF(msg, struct envoy_type_matcher_MetadataMatcher*, UPB_SIZE(0, 0), value, UPB_SIZE(4, 8), 7); +} +UPB_INLINE struct envoy_type_matcher_MetadataMatcher* envoy_config_rbac_v2_Permission_mutable_metadata(envoy_config_rbac_v2_Permission *msg, upb_arena *arena) { + struct envoy_type_matcher_MetadataMatcher* sub = (struct envoy_type_matcher_MetadataMatcher*)envoy_config_rbac_v2_Permission_metadata(msg); + if (sub == NULL) { + sub = (struct envoy_type_matcher_MetadataMatcher*)upb_msg_new(&envoy_type_matcher_MetadataMatcher_msginit, arena); + if (!sub) return NULL; + envoy_config_rbac_v2_Permission_set_metadata(msg, sub); + } + return sub; +} +UPB_INLINE void envoy_config_rbac_v2_Permission_set_not_rule(envoy_config_rbac_v2_Permission *msg, envoy_config_rbac_v2_Permission* value) { + UPB_WRITE_ONEOF(msg, envoy_config_rbac_v2_Permission*, UPB_SIZE(0, 0), value, UPB_SIZE(4, 8), 8); +} +UPB_INLINE struct envoy_config_rbac_v2_Permission* envoy_config_rbac_v2_Permission_mutable_not_rule(envoy_config_rbac_v2_Permission *msg, upb_arena *arena) { + struct envoy_config_rbac_v2_Permission* sub = (struct envoy_config_rbac_v2_Permission*)envoy_config_rbac_v2_Permission_not_rule(msg); + if (sub == NULL) { + sub = (struct envoy_config_rbac_v2_Permission*)upb_msg_new(&envoy_config_rbac_v2_Permission_msginit, arena); + if (!sub) return NULL; + envoy_config_rbac_v2_Permission_set_not_rule(msg, sub); + } + return sub; +} +UPB_INLINE void envoy_config_rbac_v2_Permission_set_requested_server_name(envoy_config_rbac_v2_Permission *msg, struct envoy_type_matcher_StringMatcher* value) { + UPB_WRITE_ONEOF(msg, struct envoy_type_matcher_StringMatcher*, UPB_SIZE(0, 0), value, UPB_SIZE(4, 8), 9); +} +UPB_INLINE struct envoy_type_matcher_StringMatcher* envoy_config_rbac_v2_Permission_mutable_requested_server_name(envoy_config_rbac_v2_Permission *msg, upb_arena *arena) { + struct envoy_type_matcher_StringMatcher* sub = (struct envoy_type_matcher_StringMatcher*)envoy_config_rbac_v2_Permission_requested_server_name(msg); + if (sub == NULL) { + sub = (struct envoy_type_matcher_StringMatcher*)upb_msg_new(&envoy_type_matcher_StringMatcher_msginit, arena); + if (!sub) return NULL; + envoy_config_rbac_v2_Permission_set_requested_server_name(msg, sub); + } + return sub; +} +UPB_INLINE void envoy_config_rbac_v2_Permission_set_url_path(envoy_config_rbac_v2_Permission *msg, struct envoy_type_matcher_PathMatcher* value) { + UPB_WRITE_ONEOF(msg, struct envoy_type_matcher_PathMatcher*, UPB_SIZE(0, 0), value, UPB_SIZE(4, 8), 10); +} +UPB_INLINE struct envoy_type_matcher_PathMatcher* envoy_config_rbac_v2_Permission_mutable_url_path(envoy_config_rbac_v2_Permission *msg, upb_arena *arena) { + struct envoy_type_matcher_PathMatcher* sub = (struct envoy_type_matcher_PathMatcher*)envoy_config_rbac_v2_Permission_url_path(msg); + if (sub == NULL) { + sub = (struct envoy_type_matcher_PathMatcher*)upb_msg_new(&envoy_type_matcher_PathMatcher_msginit, arena); + if (!sub) return NULL; + envoy_config_rbac_v2_Permission_set_url_path(msg, sub); + } + return sub; +} + +/* envoy.config.rbac.v2.Permission.Set */ + +UPB_INLINE envoy_config_rbac_v2_Permission_Set *envoy_config_rbac_v2_Permission_Set_new(upb_arena *arena) { + return (envoy_config_rbac_v2_Permission_Set *)upb_msg_new(&envoy_config_rbac_v2_Permission_Set_msginit, arena); +} +UPB_INLINE envoy_config_rbac_v2_Permission_Set *envoy_config_rbac_v2_Permission_Set_parse(const char *buf, size_t size, + upb_arena *arena) { + envoy_config_rbac_v2_Permission_Set *ret = envoy_config_rbac_v2_Permission_Set_new(arena); + return (ret && upb_decode(buf, size, ret, &envoy_config_rbac_v2_Permission_Set_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *envoy_config_rbac_v2_Permission_Set_serialize(const envoy_config_rbac_v2_Permission_Set *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &envoy_config_rbac_v2_Permission_Set_msginit, arena, len); +} + +UPB_INLINE const envoy_config_rbac_v2_Permission* const* envoy_config_rbac_v2_Permission_Set_rules(const envoy_config_rbac_v2_Permission_Set *msg, size_t *len) { return (const envoy_config_rbac_v2_Permission* const*)_upb_array_accessor(msg, UPB_SIZE(0, 0), len); } + +UPB_INLINE envoy_config_rbac_v2_Permission** envoy_config_rbac_v2_Permission_Set_mutable_rules(envoy_config_rbac_v2_Permission_Set *msg, size_t *len) { + return (envoy_config_rbac_v2_Permission**)_upb_array_mutable_accessor(msg, UPB_SIZE(0, 0), len); +} +UPB_INLINE envoy_config_rbac_v2_Permission** envoy_config_rbac_v2_Permission_Set_resize_rules(envoy_config_rbac_v2_Permission_Set *msg, size_t len, upb_arena *arena) { + return (envoy_config_rbac_v2_Permission**)_upb_array_resize_accessor(msg, UPB_SIZE(0, 0), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct envoy_config_rbac_v2_Permission* envoy_config_rbac_v2_Permission_Set_add_rules(envoy_config_rbac_v2_Permission_Set *msg, upb_arena *arena) { + struct envoy_config_rbac_v2_Permission* sub = (struct envoy_config_rbac_v2_Permission*)upb_msg_new(&envoy_config_rbac_v2_Permission_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(0, 0), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} + +/* envoy.config.rbac.v2.Principal */ + +UPB_INLINE envoy_config_rbac_v2_Principal *envoy_config_rbac_v2_Principal_new(upb_arena *arena) { + return (envoy_config_rbac_v2_Principal *)upb_msg_new(&envoy_config_rbac_v2_Principal_msginit, arena); +} +UPB_INLINE envoy_config_rbac_v2_Principal *envoy_config_rbac_v2_Principal_parse(const char *buf, size_t size, + upb_arena *arena) { + envoy_config_rbac_v2_Principal *ret = envoy_config_rbac_v2_Principal_new(arena); + return (ret && upb_decode(buf, size, ret, &envoy_config_rbac_v2_Principal_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *envoy_config_rbac_v2_Principal_serialize(const envoy_config_rbac_v2_Principal *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &envoy_config_rbac_v2_Principal_msginit, arena, len); +} + +typedef enum { + envoy_config_rbac_v2_Principal_identifier_and_ids = 1, + envoy_config_rbac_v2_Principal_identifier_or_ids = 2, + envoy_config_rbac_v2_Principal_identifier_any = 3, + envoy_config_rbac_v2_Principal_identifier_authenticated = 4, + envoy_config_rbac_v2_Principal_identifier_source_ip = 5, + envoy_config_rbac_v2_Principal_identifier_direct_remote_ip = 10, + envoy_config_rbac_v2_Principal_identifier_remote_ip = 11, + envoy_config_rbac_v2_Principal_identifier_header = 6, + envoy_config_rbac_v2_Principal_identifier_url_path = 9, + envoy_config_rbac_v2_Principal_identifier_metadata = 7, + envoy_config_rbac_v2_Principal_identifier_not_id = 8, + envoy_config_rbac_v2_Principal_identifier_NOT_SET = 0 +} envoy_config_rbac_v2_Principal_identifier_oneofcases; +UPB_INLINE envoy_config_rbac_v2_Principal_identifier_oneofcases envoy_config_rbac_v2_Principal_identifier_case(const envoy_config_rbac_v2_Principal* msg) { return (envoy_config_rbac_v2_Principal_identifier_oneofcases)UPB_FIELD_AT(msg, int32_t, UPB_SIZE(4, 8)); } + +UPB_INLINE bool envoy_config_rbac_v2_Principal_has_and_ids(const envoy_config_rbac_v2_Principal *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(4, 8), 1); } +UPB_INLINE const envoy_config_rbac_v2_Principal_Set* envoy_config_rbac_v2_Principal_and_ids(const envoy_config_rbac_v2_Principal *msg) { return UPB_READ_ONEOF(msg, const envoy_config_rbac_v2_Principal_Set*, UPB_SIZE(0, 0), UPB_SIZE(4, 8), 1, NULL); } +UPB_INLINE bool envoy_config_rbac_v2_Principal_has_or_ids(const envoy_config_rbac_v2_Principal *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(4, 8), 2); } +UPB_INLINE const envoy_config_rbac_v2_Principal_Set* envoy_config_rbac_v2_Principal_or_ids(const envoy_config_rbac_v2_Principal *msg) { return UPB_READ_ONEOF(msg, const envoy_config_rbac_v2_Principal_Set*, UPB_SIZE(0, 0), UPB_SIZE(4, 8), 2, NULL); } +UPB_INLINE bool envoy_config_rbac_v2_Principal_has_any(const envoy_config_rbac_v2_Principal *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(4, 8), 3); } +UPB_INLINE bool envoy_config_rbac_v2_Principal_any(const envoy_config_rbac_v2_Principal *msg) { return UPB_READ_ONEOF(msg, bool, UPB_SIZE(0, 0), UPB_SIZE(4, 8), 3, false); } +UPB_INLINE bool envoy_config_rbac_v2_Principal_has_authenticated(const envoy_config_rbac_v2_Principal *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(4, 8), 4); } +UPB_INLINE const envoy_config_rbac_v2_Principal_Authenticated* envoy_config_rbac_v2_Principal_authenticated(const envoy_config_rbac_v2_Principal *msg) { return UPB_READ_ONEOF(msg, const envoy_config_rbac_v2_Principal_Authenticated*, UPB_SIZE(0, 0), UPB_SIZE(4, 8), 4, NULL); } +UPB_INLINE bool envoy_config_rbac_v2_Principal_has_source_ip(const envoy_config_rbac_v2_Principal *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(4, 8), 5); } +UPB_INLINE const struct envoy_api_v2_core_CidrRange* envoy_config_rbac_v2_Principal_source_ip(const envoy_config_rbac_v2_Principal *msg) { return UPB_READ_ONEOF(msg, const struct envoy_api_v2_core_CidrRange*, UPB_SIZE(0, 0), UPB_SIZE(4, 8), 5, NULL); } +UPB_INLINE bool envoy_config_rbac_v2_Principal_has_header(const envoy_config_rbac_v2_Principal *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(4, 8), 6); } +UPB_INLINE const struct envoy_api_v2_route_HeaderMatcher* envoy_config_rbac_v2_Principal_header(const envoy_config_rbac_v2_Principal *msg) { return UPB_READ_ONEOF(msg, const struct envoy_api_v2_route_HeaderMatcher*, UPB_SIZE(0, 0), UPB_SIZE(4, 8), 6, NULL); } +UPB_INLINE bool envoy_config_rbac_v2_Principal_has_metadata(const envoy_config_rbac_v2_Principal *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(4, 8), 7); } +UPB_INLINE const struct envoy_type_matcher_MetadataMatcher* envoy_config_rbac_v2_Principal_metadata(const envoy_config_rbac_v2_Principal *msg) { return UPB_READ_ONEOF(msg, const struct envoy_type_matcher_MetadataMatcher*, UPB_SIZE(0, 0), UPB_SIZE(4, 8), 7, NULL); } +UPB_INLINE bool envoy_config_rbac_v2_Principal_has_not_id(const envoy_config_rbac_v2_Principal *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(4, 8), 8); } +UPB_INLINE const envoy_config_rbac_v2_Principal* envoy_config_rbac_v2_Principal_not_id(const envoy_config_rbac_v2_Principal *msg) { return UPB_READ_ONEOF(msg, const envoy_config_rbac_v2_Principal*, UPB_SIZE(0, 0), UPB_SIZE(4, 8), 8, NULL); } +UPB_INLINE bool envoy_config_rbac_v2_Principal_has_url_path(const envoy_config_rbac_v2_Principal *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(4, 8), 9); } +UPB_INLINE const struct envoy_type_matcher_PathMatcher* envoy_config_rbac_v2_Principal_url_path(const envoy_config_rbac_v2_Principal *msg) { return UPB_READ_ONEOF(msg, const struct envoy_type_matcher_PathMatcher*, UPB_SIZE(0, 0), UPB_SIZE(4, 8), 9, NULL); } +UPB_INLINE bool envoy_config_rbac_v2_Principal_has_direct_remote_ip(const envoy_config_rbac_v2_Principal *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(4, 8), 10); } +UPB_INLINE const struct envoy_api_v2_core_CidrRange* envoy_config_rbac_v2_Principal_direct_remote_ip(const envoy_config_rbac_v2_Principal *msg) { return UPB_READ_ONEOF(msg, const struct envoy_api_v2_core_CidrRange*, UPB_SIZE(0, 0), UPB_SIZE(4, 8), 10, NULL); } +UPB_INLINE bool envoy_config_rbac_v2_Principal_has_remote_ip(const envoy_config_rbac_v2_Principal *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(4, 8), 11); } +UPB_INLINE const struct envoy_api_v2_core_CidrRange* envoy_config_rbac_v2_Principal_remote_ip(const envoy_config_rbac_v2_Principal *msg) { return UPB_READ_ONEOF(msg, const struct envoy_api_v2_core_CidrRange*, UPB_SIZE(0, 0), UPB_SIZE(4, 8), 11, NULL); } + +UPB_INLINE void envoy_config_rbac_v2_Principal_set_and_ids(envoy_config_rbac_v2_Principal *msg, envoy_config_rbac_v2_Principal_Set* value) { + UPB_WRITE_ONEOF(msg, envoy_config_rbac_v2_Principal_Set*, UPB_SIZE(0, 0), value, UPB_SIZE(4, 8), 1); +} +UPB_INLINE struct envoy_config_rbac_v2_Principal_Set* envoy_config_rbac_v2_Principal_mutable_and_ids(envoy_config_rbac_v2_Principal *msg, upb_arena *arena) { + struct envoy_config_rbac_v2_Principal_Set* sub = (struct envoy_config_rbac_v2_Principal_Set*)envoy_config_rbac_v2_Principal_and_ids(msg); + if (sub == NULL) { + sub = (struct envoy_config_rbac_v2_Principal_Set*)upb_msg_new(&envoy_config_rbac_v2_Principal_Set_msginit, arena); + if (!sub) return NULL; + envoy_config_rbac_v2_Principal_set_and_ids(msg, sub); + } + return sub; +} +UPB_INLINE void envoy_config_rbac_v2_Principal_set_or_ids(envoy_config_rbac_v2_Principal *msg, envoy_config_rbac_v2_Principal_Set* value) { + UPB_WRITE_ONEOF(msg, envoy_config_rbac_v2_Principal_Set*, UPB_SIZE(0, 0), value, UPB_SIZE(4, 8), 2); +} +UPB_INLINE struct envoy_config_rbac_v2_Principal_Set* envoy_config_rbac_v2_Principal_mutable_or_ids(envoy_config_rbac_v2_Principal *msg, upb_arena *arena) { + struct envoy_config_rbac_v2_Principal_Set* sub = (struct envoy_config_rbac_v2_Principal_Set*)envoy_config_rbac_v2_Principal_or_ids(msg); + if (sub == NULL) { + sub = (struct envoy_config_rbac_v2_Principal_Set*)upb_msg_new(&envoy_config_rbac_v2_Principal_Set_msginit, arena); + if (!sub) return NULL; + envoy_config_rbac_v2_Principal_set_or_ids(msg, sub); + } + return sub; +} +UPB_INLINE void envoy_config_rbac_v2_Principal_set_any(envoy_config_rbac_v2_Principal *msg, bool value) { + UPB_WRITE_ONEOF(msg, bool, UPB_SIZE(0, 0), value, UPB_SIZE(4, 8), 3); +} +UPB_INLINE void envoy_config_rbac_v2_Principal_set_authenticated(envoy_config_rbac_v2_Principal *msg, envoy_config_rbac_v2_Principal_Authenticated* value) { + UPB_WRITE_ONEOF(msg, envoy_config_rbac_v2_Principal_Authenticated*, UPB_SIZE(0, 0), value, UPB_SIZE(4, 8), 4); +} +UPB_INLINE struct envoy_config_rbac_v2_Principal_Authenticated* envoy_config_rbac_v2_Principal_mutable_authenticated(envoy_config_rbac_v2_Principal *msg, upb_arena *arena) { + struct envoy_config_rbac_v2_Principal_Authenticated* sub = (struct envoy_config_rbac_v2_Principal_Authenticated*)envoy_config_rbac_v2_Principal_authenticated(msg); + if (sub == NULL) { + sub = (struct envoy_config_rbac_v2_Principal_Authenticated*)upb_msg_new(&envoy_config_rbac_v2_Principal_Authenticated_msginit, arena); + if (!sub) return NULL; + envoy_config_rbac_v2_Principal_set_authenticated(msg, sub); + } + return sub; +} +UPB_INLINE void envoy_config_rbac_v2_Principal_set_source_ip(envoy_config_rbac_v2_Principal *msg, struct envoy_api_v2_core_CidrRange* value) { + UPB_WRITE_ONEOF(msg, struct envoy_api_v2_core_CidrRange*, UPB_SIZE(0, 0), value, UPB_SIZE(4, 8), 5); +} +UPB_INLINE struct envoy_api_v2_core_CidrRange* envoy_config_rbac_v2_Principal_mutable_source_ip(envoy_config_rbac_v2_Principal *msg, upb_arena *arena) { + struct envoy_api_v2_core_CidrRange* sub = (struct envoy_api_v2_core_CidrRange*)envoy_config_rbac_v2_Principal_source_ip(msg); + if (sub == NULL) { + sub = (struct envoy_api_v2_core_CidrRange*)upb_msg_new(&envoy_api_v2_core_CidrRange_msginit, arena); + if (!sub) return NULL; + envoy_config_rbac_v2_Principal_set_source_ip(msg, sub); + } + return sub; +} +UPB_INLINE void envoy_config_rbac_v2_Principal_set_header(envoy_config_rbac_v2_Principal *msg, struct envoy_api_v2_route_HeaderMatcher* value) { + UPB_WRITE_ONEOF(msg, struct envoy_api_v2_route_HeaderMatcher*, UPB_SIZE(0, 0), value, UPB_SIZE(4, 8), 6); +} +UPB_INLINE struct envoy_api_v2_route_HeaderMatcher* envoy_config_rbac_v2_Principal_mutable_header(envoy_config_rbac_v2_Principal *msg, upb_arena *arena) { + struct envoy_api_v2_route_HeaderMatcher* sub = (struct envoy_api_v2_route_HeaderMatcher*)envoy_config_rbac_v2_Principal_header(msg); + if (sub == NULL) { + sub = (struct envoy_api_v2_route_HeaderMatcher*)upb_msg_new(&envoy_api_v2_route_HeaderMatcher_msginit, arena); + if (!sub) return NULL; + envoy_config_rbac_v2_Principal_set_header(msg, sub); + } + return sub; +} +UPB_INLINE void envoy_config_rbac_v2_Principal_set_metadata(envoy_config_rbac_v2_Principal *msg, struct envoy_type_matcher_MetadataMatcher* value) { + UPB_WRITE_ONEOF(msg, struct envoy_type_matcher_MetadataMatcher*, UPB_SIZE(0, 0), value, UPB_SIZE(4, 8), 7); +} +UPB_INLINE struct envoy_type_matcher_MetadataMatcher* envoy_config_rbac_v2_Principal_mutable_metadata(envoy_config_rbac_v2_Principal *msg, upb_arena *arena) { + struct envoy_type_matcher_MetadataMatcher* sub = (struct envoy_type_matcher_MetadataMatcher*)envoy_config_rbac_v2_Principal_metadata(msg); + if (sub == NULL) { + sub = (struct envoy_type_matcher_MetadataMatcher*)upb_msg_new(&envoy_type_matcher_MetadataMatcher_msginit, arena); + if (!sub) return NULL; + envoy_config_rbac_v2_Principal_set_metadata(msg, sub); + } + return sub; +} +UPB_INLINE void envoy_config_rbac_v2_Principal_set_not_id(envoy_config_rbac_v2_Principal *msg, envoy_config_rbac_v2_Principal* value) { + UPB_WRITE_ONEOF(msg, envoy_config_rbac_v2_Principal*, UPB_SIZE(0, 0), value, UPB_SIZE(4, 8), 8); +} +UPB_INLINE struct envoy_config_rbac_v2_Principal* envoy_config_rbac_v2_Principal_mutable_not_id(envoy_config_rbac_v2_Principal *msg, upb_arena *arena) { + struct envoy_config_rbac_v2_Principal* sub = (struct envoy_config_rbac_v2_Principal*)envoy_config_rbac_v2_Principal_not_id(msg); + if (sub == NULL) { + sub = (struct envoy_config_rbac_v2_Principal*)upb_msg_new(&envoy_config_rbac_v2_Principal_msginit, arena); + if (!sub) return NULL; + envoy_config_rbac_v2_Principal_set_not_id(msg, sub); + } + return sub; +} +UPB_INLINE void envoy_config_rbac_v2_Principal_set_url_path(envoy_config_rbac_v2_Principal *msg, struct envoy_type_matcher_PathMatcher* value) { + UPB_WRITE_ONEOF(msg, struct envoy_type_matcher_PathMatcher*, UPB_SIZE(0, 0), value, UPB_SIZE(4, 8), 9); +} +UPB_INLINE struct envoy_type_matcher_PathMatcher* envoy_config_rbac_v2_Principal_mutable_url_path(envoy_config_rbac_v2_Principal *msg, upb_arena *arena) { + struct envoy_type_matcher_PathMatcher* sub = (struct envoy_type_matcher_PathMatcher*)envoy_config_rbac_v2_Principal_url_path(msg); + if (sub == NULL) { + sub = (struct envoy_type_matcher_PathMatcher*)upb_msg_new(&envoy_type_matcher_PathMatcher_msginit, arena); + if (!sub) return NULL; + envoy_config_rbac_v2_Principal_set_url_path(msg, sub); + } + return sub; +} +UPB_INLINE void envoy_config_rbac_v2_Principal_set_direct_remote_ip(envoy_config_rbac_v2_Principal *msg, struct envoy_api_v2_core_CidrRange* value) { + UPB_WRITE_ONEOF(msg, struct envoy_api_v2_core_CidrRange*, UPB_SIZE(0, 0), value, UPB_SIZE(4, 8), 10); +} +UPB_INLINE struct envoy_api_v2_core_CidrRange* envoy_config_rbac_v2_Principal_mutable_direct_remote_ip(envoy_config_rbac_v2_Principal *msg, upb_arena *arena) { + struct envoy_api_v2_core_CidrRange* sub = (struct envoy_api_v2_core_CidrRange*)envoy_config_rbac_v2_Principal_direct_remote_ip(msg); + if (sub == NULL) { + sub = (struct envoy_api_v2_core_CidrRange*)upb_msg_new(&envoy_api_v2_core_CidrRange_msginit, arena); + if (!sub) return NULL; + envoy_config_rbac_v2_Principal_set_direct_remote_ip(msg, sub); + } + return sub; +} +UPB_INLINE void envoy_config_rbac_v2_Principal_set_remote_ip(envoy_config_rbac_v2_Principal *msg, struct envoy_api_v2_core_CidrRange* value) { + UPB_WRITE_ONEOF(msg, struct envoy_api_v2_core_CidrRange*, UPB_SIZE(0, 0), value, UPB_SIZE(4, 8), 11); +} +UPB_INLINE struct envoy_api_v2_core_CidrRange* envoy_config_rbac_v2_Principal_mutable_remote_ip(envoy_config_rbac_v2_Principal *msg, upb_arena *arena) { + struct envoy_api_v2_core_CidrRange* sub = (struct envoy_api_v2_core_CidrRange*)envoy_config_rbac_v2_Principal_remote_ip(msg); + if (sub == NULL) { + sub = (struct envoy_api_v2_core_CidrRange*)upb_msg_new(&envoy_api_v2_core_CidrRange_msginit, arena); + if (!sub) return NULL; + envoy_config_rbac_v2_Principal_set_remote_ip(msg, sub); + } + return sub; +} + +/* envoy.config.rbac.v2.Principal.Set */ + +UPB_INLINE envoy_config_rbac_v2_Principal_Set *envoy_config_rbac_v2_Principal_Set_new(upb_arena *arena) { + return (envoy_config_rbac_v2_Principal_Set *)upb_msg_new(&envoy_config_rbac_v2_Principal_Set_msginit, arena); +} +UPB_INLINE envoy_config_rbac_v2_Principal_Set *envoy_config_rbac_v2_Principal_Set_parse(const char *buf, size_t size, + upb_arena *arena) { + envoy_config_rbac_v2_Principal_Set *ret = envoy_config_rbac_v2_Principal_Set_new(arena); + return (ret && upb_decode(buf, size, ret, &envoy_config_rbac_v2_Principal_Set_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *envoy_config_rbac_v2_Principal_Set_serialize(const envoy_config_rbac_v2_Principal_Set *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &envoy_config_rbac_v2_Principal_Set_msginit, arena, len); +} + +UPB_INLINE const envoy_config_rbac_v2_Principal* const* envoy_config_rbac_v2_Principal_Set_ids(const envoy_config_rbac_v2_Principal_Set *msg, size_t *len) { return (const envoy_config_rbac_v2_Principal* const*)_upb_array_accessor(msg, UPB_SIZE(0, 0), len); } + +UPB_INLINE envoy_config_rbac_v2_Principal** envoy_config_rbac_v2_Principal_Set_mutable_ids(envoy_config_rbac_v2_Principal_Set *msg, size_t *len) { + return (envoy_config_rbac_v2_Principal**)_upb_array_mutable_accessor(msg, UPB_SIZE(0, 0), len); +} +UPB_INLINE envoy_config_rbac_v2_Principal** envoy_config_rbac_v2_Principal_Set_resize_ids(envoy_config_rbac_v2_Principal_Set *msg, size_t len, upb_arena *arena) { + return (envoy_config_rbac_v2_Principal**)_upb_array_resize_accessor(msg, UPB_SIZE(0, 0), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct envoy_config_rbac_v2_Principal* envoy_config_rbac_v2_Principal_Set_add_ids(envoy_config_rbac_v2_Principal_Set *msg, upb_arena *arena) { + struct envoy_config_rbac_v2_Principal* sub = (struct envoy_config_rbac_v2_Principal*)upb_msg_new(&envoy_config_rbac_v2_Principal_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(0, 0), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} + +/* envoy.config.rbac.v2.Principal.Authenticated */ + +UPB_INLINE envoy_config_rbac_v2_Principal_Authenticated *envoy_config_rbac_v2_Principal_Authenticated_new(upb_arena *arena) { + return (envoy_config_rbac_v2_Principal_Authenticated *)upb_msg_new(&envoy_config_rbac_v2_Principal_Authenticated_msginit, arena); +} +UPB_INLINE envoy_config_rbac_v2_Principal_Authenticated *envoy_config_rbac_v2_Principal_Authenticated_parse(const char *buf, size_t size, + upb_arena *arena) { + envoy_config_rbac_v2_Principal_Authenticated *ret = envoy_config_rbac_v2_Principal_Authenticated_new(arena); + return (ret && upb_decode(buf, size, ret, &envoy_config_rbac_v2_Principal_Authenticated_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *envoy_config_rbac_v2_Principal_Authenticated_serialize(const envoy_config_rbac_v2_Principal_Authenticated *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &envoy_config_rbac_v2_Principal_Authenticated_msginit, arena, len); +} + +UPB_INLINE const struct envoy_type_matcher_StringMatcher* envoy_config_rbac_v2_Principal_Authenticated_principal_name(const envoy_config_rbac_v2_Principal_Authenticated *msg) { return UPB_FIELD_AT(msg, const struct envoy_type_matcher_StringMatcher*, UPB_SIZE(0, 0)); } + +UPB_INLINE void envoy_config_rbac_v2_Principal_Authenticated_set_principal_name(envoy_config_rbac_v2_Principal_Authenticated *msg, struct envoy_type_matcher_StringMatcher* value) { + UPB_FIELD_AT(msg, struct envoy_type_matcher_StringMatcher*, UPB_SIZE(0, 0)) = value; +} +UPB_INLINE struct envoy_type_matcher_StringMatcher* envoy_config_rbac_v2_Principal_Authenticated_mutable_principal_name(envoy_config_rbac_v2_Principal_Authenticated *msg, upb_arena *arena) { + struct envoy_type_matcher_StringMatcher* sub = (struct envoy_type_matcher_StringMatcher*)envoy_config_rbac_v2_Principal_Authenticated_principal_name(msg); + if (sub == NULL) { + sub = (struct envoy_type_matcher_StringMatcher*)upb_msg_new(&envoy_type_matcher_StringMatcher_msginit, arena); + if (!sub) return NULL; + envoy_config_rbac_v2_Principal_Authenticated_set_principal_name(msg, sub); + } + return sub; +} + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#include "upb/port_undef.inc" + +#endif /* ENVOY_CONFIG_RBAC_V2_RBAC_PROTO_UPB_H_ */ diff --git a/src/core/ext/upb-generated/envoy/type/matcher/path.upb.c b/src/core/ext/upb-generated/envoy/type/matcher/path.upb.c new file mode 100644 index 00000000000..94d6a252410 --- /dev/null +++ b/src/core/ext/upb-generated/envoy/type/matcher/path.upb.c @@ -0,0 +1,33 @@ +/* This file was generated by upbc (the upb compiler) from the input + * file: + * + * envoy/type/matcher/path.proto + * + * Do not edit -- your changes will be discarded when the file is + * regenerated. */ + +#include +#include "upb/msg.h" +#include "envoy/type/matcher/path.upb.h" +#include "envoy/type/matcher/string.upb.h" +#include "udpa/annotations/status.upb.h" +#include "validate/validate.upb.h" + +#include "upb/port_def.inc" + +static const upb_msglayout *const envoy_type_matcher_PathMatcher_submsgs[1] = { + &envoy_type_matcher_StringMatcher_msginit, +}; + +static const upb_msglayout_field envoy_type_matcher_PathMatcher__fields[1] = { + {1, UPB_SIZE(0, 0), UPB_SIZE(-5, -9), 0, 11, 1}, +}; + +const upb_msglayout envoy_type_matcher_PathMatcher_msginit = { + &envoy_type_matcher_PathMatcher_submsgs[0], + &envoy_type_matcher_PathMatcher__fields[0], + UPB_SIZE(8, 16), 1, false, +}; + +#include "upb/port_undef.inc" + diff --git a/src/core/ext/upb-generated/envoy/type/matcher/path.upb.h b/src/core/ext/upb-generated/envoy/type/matcher/path.upb.h new file mode 100644 index 00000000000..47655a2aa99 --- /dev/null +++ b/src/core/ext/upb-generated/envoy/type/matcher/path.upb.h @@ -0,0 +1,72 @@ +/* This file was generated by upbc (the upb compiler) from the input + * file: + * + * envoy/type/matcher/path.proto + * + * Do not edit -- your changes will be discarded when the file is + * regenerated. */ + +#ifndef ENVOY_TYPE_MATCHER_PATH_PROTO_UPB_H_ +#define ENVOY_TYPE_MATCHER_PATH_PROTO_UPB_H_ + +#include "upb/generated_util.h" +#include "upb/msg.h" +#include "upb/decode.h" +#include "upb/encode.h" + +#include "upb/port_def.inc" + +#ifdef __cplusplus +extern "C" { +#endif + +struct envoy_type_matcher_PathMatcher; +typedef struct envoy_type_matcher_PathMatcher envoy_type_matcher_PathMatcher; +extern const upb_msglayout envoy_type_matcher_PathMatcher_msginit; +struct envoy_type_matcher_StringMatcher; +extern const upb_msglayout envoy_type_matcher_StringMatcher_msginit; + + +/* envoy.type.matcher.PathMatcher */ + +UPB_INLINE envoy_type_matcher_PathMatcher *envoy_type_matcher_PathMatcher_new(upb_arena *arena) { + return (envoy_type_matcher_PathMatcher *)upb_msg_new(&envoy_type_matcher_PathMatcher_msginit, arena); +} +UPB_INLINE envoy_type_matcher_PathMatcher *envoy_type_matcher_PathMatcher_parse(const char *buf, size_t size, + upb_arena *arena) { + envoy_type_matcher_PathMatcher *ret = envoy_type_matcher_PathMatcher_new(arena); + return (ret && upb_decode(buf, size, ret, &envoy_type_matcher_PathMatcher_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *envoy_type_matcher_PathMatcher_serialize(const envoy_type_matcher_PathMatcher *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &envoy_type_matcher_PathMatcher_msginit, arena, len); +} + +typedef enum { + envoy_type_matcher_PathMatcher_rule_path = 1, + envoy_type_matcher_PathMatcher_rule_NOT_SET = 0 +} envoy_type_matcher_PathMatcher_rule_oneofcases; +UPB_INLINE envoy_type_matcher_PathMatcher_rule_oneofcases envoy_type_matcher_PathMatcher_rule_case(const envoy_type_matcher_PathMatcher* msg) { return (envoy_type_matcher_PathMatcher_rule_oneofcases)UPB_FIELD_AT(msg, int32_t, UPB_SIZE(4, 8)); } + +UPB_INLINE bool envoy_type_matcher_PathMatcher_has_path(const envoy_type_matcher_PathMatcher *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(4, 8), 1); } +UPB_INLINE const struct envoy_type_matcher_StringMatcher* envoy_type_matcher_PathMatcher_path(const envoy_type_matcher_PathMatcher *msg) { return UPB_READ_ONEOF(msg, const struct envoy_type_matcher_StringMatcher*, UPB_SIZE(0, 0), UPB_SIZE(4, 8), 1, NULL); } + +UPB_INLINE void envoy_type_matcher_PathMatcher_set_path(envoy_type_matcher_PathMatcher *msg, struct envoy_type_matcher_StringMatcher* value) { + UPB_WRITE_ONEOF(msg, struct envoy_type_matcher_StringMatcher*, UPB_SIZE(0, 0), value, UPB_SIZE(4, 8), 1); +} +UPB_INLINE struct envoy_type_matcher_StringMatcher* envoy_type_matcher_PathMatcher_mutable_path(envoy_type_matcher_PathMatcher *msg, upb_arena *arena) { + struct envoy_type_matcher_StringMatcher* sub = (struct envoy_type_matcher_StringMatcher*)envoy_type_matcher_PathMatcher_path(msg); + if (sub == NULL) { + sub = (struct envoy_type_matcher_StringMatcher*)upb_msg_new(&envoy_type_matcher_StringMatcher_msginit, arena); + if (!sub) return NULL; + envoy_type_matcher_PathMatcher_set_path(msg, sub); + } + return sub; +} + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#include "upb/port_undef.inc" + +#endif /* ENVOY_TYPE_MATCHER_PATH_PROTO_UPB_H_ */ diff --git a/src/core/ext/upb-generated/google/api/expr/v1alpha1/syntax.upb.c b/src/core/ext/upb-generated/google/api/expr/v1alpha1/syntax.upb.c new file mode 100644 index 00000000000..ba15ac0b501 --- /dev/null +++ b/src/core/ext/upb-generated/google/api/expr/v1alpha1/syntax.upb.c @@ -0,0 +1,234 @@ +/* This file was generated by upbc (the upb compiler) from the input + * file: + * + * google/api/expr/v1alpha1/syntax.proto + * + * Do not edit -- your changes will be discarded when the file is + * regenerated. */ + +#include +#include "upb/msg.h" +#include "google/api/expr/v1alpha1/syntax.upb.h" +#include "google/protobuf/duration.upb.h" +#include "google/protobuf/struct.upb.h" +#include "google/protobuf/timestamp.upb.h" + +#include "upb/port_def.inc" + +static const upb_msglayout *const google_api_expr_v1alpha1_ParsedExpr_submsgs[2] = { + &google_api_expr_v1alpha1_Expr_msginit, + &google_api_expr_v1alpha1_SourceInfo_msginit, +}; + +static const upb_msglayout_field google_api_expr_v1alpha1_ParsedExpr__fields[2] = { + {2, UPB_SIZE(0, 0), 0, 0, 11, 1}, + {3, UPB_SIZE(4, 8), 0, 1, 11, 1}, +}; + +const upb_msglayout google_api_expr_v1alpha1_ParsedExpr_msginit = { + &google_api_expr_v1alpha1_ParsedExpr_submsgs[0], + &google_api_expr_v1alpha1_ParsedExpr__fields[0], + UPB_SIZE(8, 16), 2, false, +}; + +static const upb_msglayout *const google_api_expr_v1alpha1_Expr_submsgs[7] = { + &google_api_expr_v1alpha1_Constant_msginit, + &google_api_expr_v1alpha1_Expr_Call_msginit, + &google_api_expr_v1alpha1_Expr_Comprehension_msginit, + &google_api_expr_v1alpha1_Expr_CreateList_msginit, + &google_api_expr_v1alpha1_Expr_CreateStruct_msginit, + &google_api_expr_v1alpha1_Expr_Ident_msginit, + &google_api_expr_v1alpha1_Expr_Select_msginit, +}; + +static const upb_msglayout_field google_api_expr_v1alpha1_Expr__fields[8] = { + {2, UPB_SIZE(0, 0), 0, 0, 3, 1}, + {3, UPB_SIZE(8, 8), UPB_SIZE(-13, -17), 0, 11, 1}, + {4, UPB_SIZE(8, 8), UPB_SIZE(-13, -17), 5, 11, 1}, + {5, UPB_SIZE(8, 8), UPB_SIZE(-13, -17), 6, 11, 1}, + {6, UPB_SIZE(8, 8), UPB_SIZE(-13, -17), 1, 11, 1}, + {7, UPB_SIZE(8, 8), UPB_SIZE(-13, -17), 3, 11, 1}, + {8, UPB_SIZE(8, 8), UPB_SIZE(-13, -17), 4, 11, 1}, + {9, UPB_SIZE(8, 8), UPB_SIZE(-13, -17), 2, 11, 1}, +}; + +const upb_msglayout google_api_expr_v1alpha1_Expr_msginit = { + &google_api_expr_v1alpha1_Expr_submsgs[0], + &google_api_expr_v1alpha1_Expr__fields[0], + UPB_SIZE(16, 24), 8, false, +}; + +static const upb_msglayout_field google_api_expr_v1alpha1_Expr_Ident__fields[1] = { + {1, UPB_SIZE(0, 0), 0, 0, 9, 1}, +}; + +const upb_msglayout google_api_expr_v1alpha1_Expr_Ident_msginit = { + NULL, + &google_api_expr_v1alpha1_Expr_Ident__fields[0], + UPB_SIZE(8, 16), 1, false, +}; + +static const upb_msglayout *const google_api_expr_v1alpha1_Expr_Select_submsgs[1] = { + &google_api_expr_v1alpha1_Expr_msginit, +}; + +static const upb_msglayout_field google_api_expr_v1alpha1_Expr_Select__fields[3] = { + {1, UPB_SIZE(12, 24), 0, 0, 11, 1}, + {2, UPB_SIZE(4, 8), 0, 0, 9, 1}, + {3, UPB_SIZE(0, 0), 0, 0, 8, 1}, +}; + +const upb_msglayout google_api_expr_v1alpha1_Expr_Select_msginit = { + &google_api_expr_v1alpha1_Expr_Select_submsgs[0], + &google_api_expr_v1alpha1_Expr_Select__fields[0], + UPB_SIZE(16, 32), 3, false, +}; + +static const upb_msglayout *const google_api_expr_v1alpha1_Expr_Call_submsgs[2] = { + &google_api_expr_v1alpha1_Expr_msginit, +}; + +static const upb_msglayout_field google_api_expr_v1alpha1_Expr_Call__fields[3] = { + {1, UPB_SIZE(8, 16), 0, 0, 11, 1}, + {2, UPB_SIZE(0, 0), 0, 0, 9, 1}, + {3, UPB_SIZE(12, 24), 0, 0, 11, 3}, +}; + +const upb_msglayout google_api_expr_v1alpha1_Expr_Call_msginit = { + &google_api_expr_v1alpha1_Expr_Call_submsgs[0], + &google_api_expr_v1alpha1_Expr_Call__fields[0], + UPB_SIZE(16, 32), 3, false, +}; + +static const upb_msglayout *const google_api_expr_v1alpha1_Expr_CreateList_submsgs[1] = { + &google_api_expr_v1alpha1_Expr_msginit, +}; + +static const upb_msglayout_field google_api_expr_v1alpha1_Expr_CreateList__fields[1] = { + {1, UPB_SIZE(0, 0), 0, 0, 11, 3}, +}; + +const upb_msglayout google_api_expr_v1alpha1_Expr_CreateList_msginit = { + &google_api_expr_v1alpha1_Expr_CreateList_submsgs[0], + &google_api_expr_v1alpha1_Expr_CreateList__fields[0], + UPB_SIZE(4, 8), 1, false, +}; + +static const upb_msglayout *const google_api_expr_v1alpha1_Expr_CreateStruct_submsgs[1] = { + &google_api_expr_v1alpha1_Expr_CreateStruct_Entry_msginit, +}; + +static const upb_msglayout_field google_api_expr_v1alpha1_Expr_CreateStruct__fields[2] = { + {1, UPB_SIZE(0, 0), 0, 0, 9, 1}, + {2, UPB_SIZE(8, 16), 0, 0, 11, 3}, +}; + +const upb_msglayout google_api_expr_v1alpha1_Expr_CreateStruct_msginit = { + &google_api_expr_v1alpha1_Expr_CreateStruct_submsgs[0], + &google_api_expr_v1alpha1_Expr_CreateStruct__fields[0], + UPB_SIZE(16, 32), 2, false, +}; + +static const upb_msglayout *const google_api_expr_v1alpha1_Expr_CreateStruct_Entry_submsgs[2] = { + &google_api_expr_v1alpha1_Expr_msginit, +}; + +static const upb_msglayout_field google_api_expr_v1alpha1_Expr_CreateStruct_Entry__fields[4] = { + {1, UPB_SIZE(0, 0), 0, 0, 3, 1}, + {2, UPB_SIZE(12, 16), UPB_SIZE(-21, -33), 0, 9, 1}, + {3, UPB_SIZE(12, 16), UPB_SIZE(-21, -33), 0, 11, 1}, + {4, UPB_SIZE(8, 8), 0, 0, 11, 1}, +}; + +const upb_msglayout google_api_expr_v1alpha1_Expr_CreateStruct_Entry_msginit = { + &google_api_expr_v1alpha1_Expr_CreateStruct_Entry_submsgs[0], + &google_api_expr_v1alpha1_Expr_CreateStruct_Entry__fields[0], + UPB_SIZE(24, 48), 4, false, +}; + +static const upb_msglayout *const google_api_expr_v1alpha1_Expr_Comprehension_submsgs[5] = { + &google_api_expr_v1alpha1_Expr_msginit, +}; + +static const upb_msglayout_field google_api_expr_v1alpha1_Expr_Comprehension__fields[7] = { + {1, UPB_SIZE(0, 0), 0, 0, 9, 1}, + {2, UPB_SIZE(16, 32), 0, 0, 11, 1}, + {3, UPB_SIZE(8, 16), 0, 0, 9, 1}, + {4, UPB_SIZE(20, 40), 0, 0, 11, 1}, + {5, UPB_SIZE(24, 48), 0, 0, 11, 1}, + {6, UPB_SIZE(28, 56), 0, 0, 11, 1}, + {7, UPB_SIZE(32, 64), 0, 0, 11, 1}, +}; + +const upb_msglayout google_api_expr_v1alpha1_Expr_Comprehension_msginit = { + &google_api_expr_v1alpha1_Expr_Comprehension_submsgs[0], + &google_api_expr_v1alpha1_Expr_Comprehension__fields[0], + UPB_SIZE(40, 80), 7, false, +}; + +static const upb_msglayout *const google_api_expr_v1alpha1_Constant_submsgs[2] = { + &google_protobuf_Duration_msginit, + &google_protobuf_Timestamp_msginit, +}; + +static const upb_msglayout_field google_api_expr_v1alpha1_Constant__fields[9] = { + {1, UPB_SIZE(0, 0), UPB_SIZE(-9, -17), 0, 14, 1}, + {2, UPB_SIZE(0, 0), UPB_SIZE(-9, -17), 0, 8, 1}, + {3, UPB_SIZE(0, 0), UPB_SIZE(-9, -17), 0, 3, 1}, + {4, UPB_SIZE(0, 0), UPB_SIZE(-9, -17), 0, 4, 1}, + {5, UPB_SIZE(0, 0), UPB_SIZE(-9, -17), 0, 1, 1}, + {6, UPB_SIZE(0, 0), UPB_SIZE(-9, -17), 0, 9, 1}, + {7, UPB_SIZE(0, 0), UPB_SIZE(-9, -17), 0, 12, 1}, + {8, UPB_SIZE(0, 0), UPB_SIZE(-9, -17), 0, 11, 1}, + {9, UPB_SIZE(0, 0), UPB_SIZE(-9, -17), 1, 11, 1}, +}; + +const upb_msglayout google_api_expr_v1alpha1_Constant_msginit = { + &google_api_expr_v1alpha1_Constant_submsgs[0], + &google_api_expr_v1alpha1_Constant__fields[0], + UPB_SIZE(16, 32), 9, false, +}; + +static const upb_msglayout *const google_api_expr_v1alpha1_SourceInfo_submsgs[1] = { + &google_api_expr_v1alpha1_SourceInfo_PositionsEntry_msginit, +}; + +static const upb_msglayout_field google_api_expr_v1alpha1_SourceInfo__fields[4] = { + {1, UPB_SIZE(0, 0), 0, 0, 9, 1}, + {2, UPB_SIZE(8, 16), 0, 0, 9, 1}, + {3, UPB_SIZE(16, 32), 0, 0, 5, 3}, + {4, UPB_SIZE(20, 40), 0, 0, 11, 3}, +}; + +const upb_msglayout google_api_expr_v1alpha1_SourceInfo_msginit = { + &google_api_expr_v1alpha1_SourceInfo_submsgs[0], + &google_api_expr_v1alpha1_SourceInfo__fields[0], + UPB_SIZE(24, 48), 4, false, +}; + +static const upb_msglayout_field google_api_expr_v1alpha1_SourceInfo_PositionsEntry__fields[2] = { + {1, UPB_SIZE(0, 0), 0, 0, 3, 1}, + {2, UPB_SIZE(8, 8), 0, 0, 5, 1}, +}; + +const upb_msglayout google_api_expr_v1alpha1_SourceInfo_PositionsEntry_msginit = { + NULL, + &google_api_expr_v1alpha1_SourceInfo_PositionsEntry__fields[0], + UPB_SIZE(16, 16), 2, false, +}; + +static const upb_msglayout_field google_api_expr_v1alpha1_SourcePosition__fields[4] = { + {1, UPB_SIZE(12, 16), 0, 0, 9, 1}, + {2, UPB_SIZE(0, 0), 0, 0, 5, 1}, + {3, UPB_SIZE(4, 4), 0, 0, 5, 1}, + {4, UPB_SIZE(8, 8), 0, 0, 5, 1}, +}; + +const upb_msglayout google_api_expr_v1alpha1_SourcePosition_msginit = { + NULL, + &google_api_expr_v1alpha1_SourcePosition__fields[0], + UPB_SIZE(24, 32), 4, false, +}; + +#include "upb/port_undef.inc" + diff --git a/src/core/ext/upb-generated/google/api/expr/v1alpha1/syntax.upb.h b/src/core/ext/upb-generated/google/api/expr/v1alpha1/syntax.upb.h new file mode 100644 index 00000000000..8a04020469b --- /dev/null +++ b/src/core/ext/upb-generated/google/api/expr/v1alpha1/syntax.upb.h @@ -0,0 +1,760 @@ +/* This file was generated by upbc (the upb compiler) from the input + * file: + * + * google/api/expr/v1alpha1/syntax.proto + * + * Do not edit -- your changes will be discarded when the file is + * regenerated. */ + +#ifndef GOOGLE_API_EXPR_V1ALPHA1_SYNTAX_PROTO_UPB_H_ +#define GOOGLE_API_EXPR_V1ALPHA1_SYNTAX_PROTO_UPB_H_ + +#include "upb/generated_util.h" +#include "upb/msg.h" +#include "upb/decode.h" +#include "upb/encode.h" + +#include "upb/port_def.inc" + +#ifdef __cplusplus +extern "C" { +#endif + +struct google_api_expr_v1alpha1_ParsedExpr; +struct google_api_expr_v1alpha1_Expr; +struct google_api_expr_v1alpha1_Expr_Ident; +struct google_api_expr_v1alpha1_Expr_Select; +struct google_api_expr_v1alpha1_Expr_Call; +struct google_api_expr_v1alpha1_Expr_CreateList; +struct google_api_expr_v1alpha1_Expr_CreateStruct; +struct google_api_expr_v1alpha1_Expr_CreateStruct_Entry; +struct google_api_expr_v1alpha1_Expr_Comprehension; +struct google_api_expr_v1alpha1_Constant; +struct google_api_expr_v1alpha1_SourceInfo; +struct google_api_expr_v1alpha1_SourceInfo_PositionsEntry; +struct google_api_expr_v1alpha1_SourcePosition; +typedef struct google_api_expr_v1alpha1_ParsedExpr google_api_expr_v1alpha1_ParsedExpr; +typedef struct google_api_expr_v1alpha1_Expr google_api_expr_v1alpha1_Expr; +typedef struct google_api_expr_v1alpha1_Expr_Ident google_api_expr_v1alpha1_Expr_Ident; +typedef struct google_api_expr_v1alpha1_Expr_Select google_api_expr_v1alpha1_Expr_Select; +typedef struct google_api_expr_v1alpha1_Expr_Call google_api_expr_v1alpha1_Expr_Call; +typedef struct google_api_expr_v1alpha1_Expr_CreateList google_api_expr_v1alpha1_Expr_CreateList; +typedef struct google_api_expr_v1alpha1_Expr_CreateStruct google_api_expr_v1alpha1_Expr_CreateStruct; +typedef struct google_api_expr_v1alpha1_Expr_CreateStruct_Entry google_api_expr_v1alpha1_Expr_CreateStruct_Entry; +typedef struct google_api_expr_v1alpha1_Expr_Comprehension google_api_expr_v1alpha1_Expr_Comprehension; +typedef struct google_api_expr_v1alpha1_Constant google_api_expr_v1alpha1_Constant; +typedef struct google_api_expr_v1alpha1_SourceInfo google_api_expr_v1alpha1_SourceInfo; +typedef struct google_api_expr_v1alpha1_SourceInfo_PositionsEntry google_api_expr_v1alpha1_SourceInfo_PositionsEntry; +typedef struct google_api_expr_v1alpha1_SourcePosition google_api_expr_v1alpha1_SourcePosition; +extern const upb_msglayout google_api_expr_v1alpha1_ParsedExpr_msginit; +extern const upb_msglayout google_api_expr_v1alpha1_Expr_msginit; +extern const upb_msglayout google_api_expr_v1alpha1_Expr_Ident_msginit; +extern const upb_msglayout google_api_expr_v1alpha1_Expr_Select_msginit; +extern const upb_msglayout google_api_expr_v1alpha1_Expr_Call_msginit; +extern const upb_msglayout google_api_expr_v1alpha1_Expr_CreateList_msginit; +extern const upb_msglayout google_api_expr_v1alpha1_Expr_CreateStruct_msginit; +extern const upb_msglayout google_api_expr_v1alpha1_Expr_CreateStruct_Entry_msginit; +extern const upb_msglayout google_api_expr_v1alpha1_Expr_Comprehension_msginit; +extern const upb_msglayout google_api_expr_v1alpha1_Constant_msginit; +extern const upb_msglayout google_api_expr_v1alpha1_SourceInfo_msginit; +extern const upb_msglayout google_api_expr_v1alpha1_SourceInfo_PositionsEntry_msginit; +extern const upb_msglayout google_api_expr_v1alpha1_SourcePosition_msginit; +struct google_protobuf_Duration; +struct google_protobuf_Timestamp; +extern const upb_msglayout google_protobuf_Duration_msginit; +extern const upb_msglayout google_protobuf_Timestamp_msginit; + + +/* google.api.expr.v1alpha1.ParsedExpr */ + +UPB_INLINE google_api_expr_v1alpha1_ParsedExpr *google_api_expr_v1alpha1_ParsedExpr_new(upb_arena *arena) { + return (google_api_expr_v1alpha1_ParsedExpr *)upb_msg_new(&google_api_expr_v1alpha1_ParsedExpr_msginit, arena); +} +UPB_INLINE google_api_expr_v1alpha1_ParsedExpr *google_api_expr_v1alpha1_ParsedExpr_parse(const char *buf, size_t size, + upb_arena *arena) { + google_api_expr_v1alpha1_ParsedExpr *ret = google_api_expr_v1alpha1_ParsedExpr_new(arena); + return (ret && upb_decode(buf, size, ret, &google_api_expr_v1alpha1_ParsedExpr_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_api_expr_v1alpha1_ParsedExpr_serialize(const google_api_expr_v1alpha1_ParsedExpr *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_api_expr_v1alpha1_ParsedExpr_msginit, arena, len); +} + +UPB_INLINE const google_api_expr_v1alpha1_Expr* google_api_expr_v1alpha1_ParsedExpr_expr(const google_api_expr_v1alpha1_ParsedExpr *msg) { return UPB_FIELD_AT(msg, const google_api_expr_v1alpha1_Expr*, UPB_SIZE(0, 0)); } +UPB_INLINE const google_api_expr_v1alpha1_SourceInfo* google_api_expr_v1alpha1_ParsedExpr_source_info(const google_api_expr_v1alpha1_ParsedExpr *msg) { return UPB_FIELD_AT(msg, const google_api_expr_v1alpha1_SourceInfo*, UPB_SIZE(4, 8)); } + +UPB_INLINE void google_api_expr_v1alpha1_ParsedExpr_set_expr(google_api_expr_v1alpha1_ParsedExpr *msg, google_api_expr_v1alpha1_Expr* value) { + UPB_FIELD_AT(msg, google_api_expr_v1alpha1_Expr*, UPB_SIZE(0, 0)) = value; +} +UPB_INLINE struct google_api_expr_v1alpha1_Expr* google_api_expr_v1alpha1_ParsedExpr_mutable_expr(google_api_expr_v1alpha1_ParsedExpr *msg, upb_arena *arena) { + struct google_api_expr_v1alpha1_Expr* sub = (struct google_api_expr_v1alpha1_Expr*)google_api_expr_v1alpha1_ParsedExpr_expr(msg); + if (sub == NULL) { + sub = (struct google_api_expr_v1alpha1_Expr*)upb_msg_new(&google_api_expr_v1alpha1_Expr_msginit, arena); + if (!sub) return NULL; + google_api_expr_v1alpha1_ParsedExpr_set_expr(msg, sub); + } + return sub; +} +UPB_INLINE void google_api_expr_v1alpha1_ParsedExpr_set_source_info(google_api_expr_v1alpha1_ParsedExpr *msg, google_api_expr_v1alpha1_SourceInfo* value) { + UPB_FIELD_AT(msg, google_api_expr_v1alpha1_SourceInfo*, UPB_SIZE(4, 8)) = value; +} +UPB_INLINE struct google_api_expr_v1alpha1_SourceInfo* google_api_expr_v1alpha1_ParsedExpr_mutable_source_info(google_api_expr_v1alpha1_ParsedExpr *msg, upb_arena *arena) { + struct google_api_expr_v1alpha1_SourceInfo* sub = (struct google_api_expr_v1alpha1_SourceInfo*)google_api_expr_v1alpha1_ParsedExpr_source_info(msg); + if (sub == NULL) { + sub = (struct google_api_expr_v1alpha1_SourceInfo*)upb_msg_new(&google_api_expr_v1alpha1_SourceInfo_msginit, arena); + if (!sub) return NULL; + google_api_expr_v1alpha1_ParsedExpr_set_source_info(msg, sub); + } + return sub; +} + +/* google.api.expr.v1alpha1.Expr */ + +UPB_INLINE google_api_expr_v1alpha1_Expr *google_api_expr_v1alpha1_Expr_new(upb_arena *arena) { + return (google_api_expr_v1alpha1_Expr *)upb_msg_new(&google_api_expr_v1alpha1_Expr_msginit, arena); +} +UPB_INLINE google_api_expr_v1alpha1_Expr *google_api_expr_v1alpha1_Expr_parse(const char *buf, size_t size, + upb_arena *arena) { + google_api_expr_v1alpha1_Expr *ret = google_api_expr_v1alpha1_Expr_new(arena); + return (ret && upb_decode(buf, size, ret, &google_api_expr_v1alpha1_Expr_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_api_expr_v1alpha1_Expr_serialize(const google_api_expr_v1alpha1_Expr *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_api_expr_v1alpha1_Expr_msginit, arena, len); +} + +typedef enum { + google_api_expr_v1alpha1_Expr_expr_kind_const_expr = 3, + google_api_expr_v1alpha1_Expr_expr_kind_ident_expr = 4, + google_api_expr_v1alpha1_Expr_expr_kind_select_expr = 5, + google_api_expr_v1alpha1_Expr_expr_kind_call_expr = 6, + google_api_expr_v1alpha1_Expr_expr_kind_list_expr = 7, + google_api_expr_v1alpha1_Expr_expr_kind_struct_expr = 8, + google_api_expr_v1alpha1_Expr_expr_kind_comprehension_expr = 9, + google_api_expr_v1alpha1_Expr_expr_kind_NOT_SET = 0 +} google_api_expr_v1alpha1_Expr_expr_kind_oneofcases; +UPB_INLINE google_api_expr_v1alpha1_Expr_expr_kind_oneofcases google_api_expr_v1alpha1_Expr_expr_kind_case(const google_api_expr_v1alpha1_Expr* msg) { return (google_api_expr_v1alpha1_Expr_expr_kind_oneofcases)UPB_FIELD_AT(msg, int32_t, UPB_SIZE(12, 16)); } + +UPB_INLINE int64_t google_api_expr_v1alpha1_Expr_id(const google_api_expr_v1alpha1_Expr *msg) { return UPB_FIELD_AT(msg, int64_t, UPB_SIZE(0, 0)); } +UPB_INLINE bool google_api_expr_v1alpha1_Expr_has_const_expr(const google_api_expr_v1alpha1_Expr *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(12, 16), 3); } +UPB_INLINE const google_api_expr_v1alpha1_Constant* google_api_expr_v1alpha1_Expr_const_expr(const google_api_expr_v1alpha1_Expr *msg) { return UPB_READ_ONEOF(msg, const google_api_expr_v1alpha1_Constant*, UPB_SIZE(8, 8), UPB_SIZE(12, 16), 3, NULL); } +UPB_INLINE bool google_api_expr_v1alpha1_Expr_has_ident_expr(const google_api_expr_v1alpha1_Expr *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(12, 16), 4); } +UPB_INLINE const google_api_expr_v1alpha1_Expr_Ident* google_api_expr_v1alpha1_Expr_ident_expr(const google_api_expr_v1alpha1_Expr *msg) { return UPB_READ_ONEOF(msg, const google_api_expr_v1alpha1_Expr_Ident*, UPB_SIZE(8, 8), UPB_SIZE(12, 16), 4, NULL); } +UPB_INLINE bool google_api_expr_v1alpha1_Expr_has_select_expr(const google_api_expr_v1alpha1_Expr *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(12, 16), 5); } +UPB_INLINE const google_api_expr_v1alpha1_Expr_Select* google_api_expr_v1alpha1_Expr_select_expr(const google_api_expr_v1alpha1_Expr *msg) { return UPB_READ_ONEOF(msg, const google_api_expr_v1alpha1_Expr_Select*, UPB_SIZE(8, 8), UPB_SIZE(12, 16), 5, NULL); } +UPB_INLINE bool google_api_expr_v1alpha1_Expr_has_call_expr(const google_api_expr_v1alpha1_Expr *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(12, 16), 6); } +UPB_INLINE const google_api_expr_v1alpha1_Expr_Call* google_api_expr_v1alpha1_Expr_call_expr(const google_api_expr_v1alpha1_Expr *msg) { return UPB_READ_ONEOF(msg, const google_api_expr_v1alpha1_Expr_Call*, UPB_SIZE(8, 8), UPB_SIZE(12, 16), 6, NULL); } +UPB_INLINE bool google_api_expr_v1alpha1_Expr_has_list_expr(const google_api_expr_v1alpha1_Expr *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(12, 16), 7); } +UPB_INLINE const google_api_expr_v1alpha1_Expr_CreateList* google_api_expr_v1alpha1_Expr_list_expr(const google_api_expr_v1alpha1_Expr *msg) { return UPB_READ_ONEOF(msg, const google_api_expr_v1alpha1_Expr_CreateList*, UPB_SIZE(8, 8), UPB_SIZE(12, 16), 7, NULL); } +UPB_INLINE bool google_api_expr_v1alpha1_Expr_has_struct_expr(const google_api_expr_v1alpha1_Expr *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(12, 16), 8); } +UPB_INLINE const google_api_expr_v1alpha1_Expr_CreateStruct* google_api_expr_v1alpha1_Expr_struct_expr(const google_api_expr_v1alpha1_Expr *msg) { return UPB_READ_ONEOF(msg, const google_api_expr_v1alpha1_Expr_CreateStruct*, UPB_SIZE(8, 8), UPB_SIZE(12, 16), 8, NULL); } +UPB_INLINE bool google_api_expr_v1alpha1_Expr_has_comprehension_expr(const google_api_expr_v1alpha1_Expr *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(12, 16), 9); } +UPB_INLINE const google_api_expr_v1alpha1_Expr_Comprehension* google_api_expr_v1alpha1_Expr_comprehension_expr(const google_api_expr_v1alpha1_Expr *msg) { return UPB_READ_ONEOF(msg, const google_api_expr_v1alpha1_Expr_Comprehension*, UPB_SIZE(8, 8), UPB_SIZE(12, 16), 9, NULL); } + +UPB_INLINE void google_api_expr_v1alpha1_Expr_set_id(google_api_expr_v1alpha1_Expr *msg, int64_t value) { + UPB_FIELD_AT(msg, int64_t, UPB_SIZE(0, 0)) = value; +} +UPB_INLINE void google_api_expr_v1alpha1_Expr_set_const_expr(google_api_expr_v1alpha1_Expr *msg, google_api_expr_v1alpha1_Constant* value) { + UPB_WRITE_ONEOF(msg, google_api_expr_v1alpha1_Constant*, UPB_SIZE(8, 8), value, UPB_SIZE(12, 16), 3); +} +UPB_INLINE struct google_api_expr_v1alpha1_Constant* google_api_expr_v1alpha1_Expr_mutable_const_expr(google_api_expr_v1alpha1_Expr *msg, upb_arena *arena) { + struct google_api_expr_v1alpha1_Constant* sub = (struct google_api_expr_v1alpha1_Constant*)google_api_expr_v1alpha1_Expr_const_expr(msg); + if (sub == NULL) { + sub = (struct google_api_expr_v1alpha1_Constant*)upb_msg_new(&google_api_expr_v1alpha1_Constant_msginit, arena); + if (!sub) return NULL; + google_api_expr_v1alpha1_Expr_set_const_expr(msg, sub); + } + return sub; +} +UPB_INLINE void google_api_expr_v1alpha1_Expr_set_ident_expr(google_api_expr_v1alpha1_Expr *msg, google_api_expr_v1alpha1_Expr_Ident* value) { + UPB_WRITE_ONEOF(msg, google_api_expr_v1alpha1_Expr_Ident*, UPB_SIZE(8, 8), value, UPB_SIZE(12, 16), 4); +} +UPB_INLINE struct google_api_expr_v1alpha1_Expr_Ident* google_api_expr_v1alpha1_Expr_mutable_ident_expr(google_api_expr_v1alpha1_Expr *msg, upb_arena *arena) { + struct google_api_expr_v1alpha1_Expr_Ident* sub = (struct google_api_expr_v1alpha1_Expr_Ident*)google_api_expr_v1alpha1_Expr_ident_expr(msg); + if (sub == NULL) { + sub = (struct google_api_expr_v1alpha1_Expr_Ident*)upb_msg_new(&google_api_expr_v1alpha1_Expr_Ident_msginit, arena); + if (!sub) return NULL; + google_api_expr_v1alpha1_Expr_set_ident_expr(msg, sub); + } + return sub; +} +UPB_INLINE void google_api_expr_v1alpha1_Expr_set_select_expr(google_api_expr_v1alpha1_Expr *msg, google_api_expr_v1alpha1_Expr_Select* value) { + UPB_WRITE_ONEOF(msg, google_api_expr_v1alpha1_Expr_Select*, UPB_SIZE(8, 8), value, UPB_SIZE(12, 16), 5); +} +UPB_INLINE struct google_api_expr_v1alpha1_Expr_Select* google_api_expr_v1alpha1_Expr_mutable_select_expr(google_api_expr_v1alpha1_Expr *msg, upb_arena *arena) { + struct google_api_expr_v1alpha1_Expr_Select* sub = (struct google_api_expr_v1alpha1_Expr_Select*)google_api_expr_v1alpha1_Expr_select_expr(msg); + if (sub == NULL) { + sub = (struct google_api_expr_v1alpha1_Expr_Select*)upb_msg_new(&google_api_expr_v1alpha1_Expr_Select_msginit, arena); + if (!sub) return NULL; + google_api_expr_v1alpha1_Expr_set_select_expr(msg, sub); + } + return sub; +} +UPB_INLINE void google_api_expr_v1alpha1_Expr_set_call_expr(google_api_expr_v1alpha1_Expr *msg, google_api_expr_v1alpha1_Expr_Call* value) { + UPB_WRITE_ONEOF(msg, google_api_expr_v1alpha1_Expr_Call*, UPB_SIZE(8, 8), value, UPB_SIZE(12, 16), 6); +} +UPB_INLINE struct google_api_expr_v1alpha1_Expr_Call* google_api_expr_v1alpha1_Expr_mutable_call_expr(google_api_expr_v1alpha1_Expr *msg, upb_arena *arena) { + struct google_api_expr_v1alpha1_Expr_Call* sub = (struct google_api_expr_v1alpha1_Expr_Call*)google_api_expr_v1alpha1_Expr_call_expr(msg); + if (sub == NULL) { + sub = (struct google_api_expr_v1alpha1_Expr_Call*)upb_msg_new(&google_api_expr_v1alpha1_Expr_Call_msginit, arena); + if (!sub) return NULL; + google_api_expr_v1alpha1_Expr_set_call_expr(msg, sub); + } + return sub; +} +UPB_INLINE void google_api_expr_v1alpha1_Expr_set_list_expr(google_api_expr_v1alpha1_Expr *msg, google_api_expr_v1alpha1_Expr_CreateList* value) { + UPB_WRITE_ONEOF(msg, google_api_expr_v1alpha1_Expr_CreateList*, UPB_SIZE(8, 8), value, UPB_SIZE(12, 16), 7); +} +UPB_INLINE struct google_api_expr_v1alpha1_Expr_CreateList* google_api_expr_v1alpha1_Expr_mutable_list_expr(google_api_expr_v1alpha1_Expr *msg, upb_arena *arena) { + struct google_api_expr_v1alpha1_Expr_CreateList* sub = (struct google_api_expr_v1alpha1_Expr_CreateList*)google_api_expr_v1alpha1_Expr_list_expr(msg); + if (sub == NULL) { + sub = (struct google_api_expr_v1alpha1_Expr_CreateList*)upb_msg_new(&google_api_expr_v1alpha1_Expr_CreateList_msginit, arena); + if (!sub) return NULL; + google_api_expr_v1alpha1_Expr_set_list_expr(msg, sub); + } + return sub; +} +UPB_INLINE void google_api_expr_v1alpha1_Expr_set_struct_expr(google_api_expr_v1alpha1_Expr *msg, google_api_expr_v1alpha1_Expr_CreateStruct* value) { + UPB_WRITE_ONEOF(msg, google_api_expr_v1alpha1_Expr_CreateStruct*, UPB_SIZE(8, 8), value, UPB_SIZE(12, 16), 8); +} +UPB_INLINE struct google_api_expr_v1alpha1_Expr_CreateStruct* google_api_expr_v1alpha1_Expr_mutable_struct_expr(google_api_expr_v1alpha1_Expr *msg, upb_arena *arena) { + struct google_api_expr_v1alpha1_Expr_CreateStruct* sub = (struct google_api_expr_v1alpha1_Expr_CreateStruct*)google_api_expr_v1alpha1_Expr_struct_expr(msg); + if (sub == NULL) { + sub = (struct google_api_expr_v1alpha1_Expr_CreateStruct*)upb_msg_new(&google_api_expr_v1alpha1_Expr_CreateStruct_msginit, arena); + if (!sub) return NULL; + google_api_expr_v1alpha1_Expr_set_struct_expr(msg, sub); + } + return sub; +} +UPB_INLINE void google_api_expr_v1alpha1_Expr_set_comprehension_expr(google_api_expr_v1alpha1_Expr *msg, google_api_expr_v1alpha1_Expr_Comprehension* value) { + UPB_WRITE_ONEOF(msg, google_api_expr_v1alpha1_Expr_Comprehension*, UPB_SIZE(8, 8), value, UPB_SIZE(12, 16), 9); +} +UPB_INLINE struct google_api_expr_v1alpha1_Expr_Comprehension* google_api_expr_v1alpha1_Expr_mutable_comprehension_expr(google_api_expr_v1alpha1_Expr *msg, upb_arena *arena) { + struct google_api_expr_v1alpha1_Expr_Comprehension* sub = (struct google_api_expr_v1alpha1_Expr_Comprehension*)google_api_expr_v1alpha1_Expr_comprehension_expr(msg); + if (sub == NULL) { + sub = (struct google_api_expr_v1alpha1_Expr_Comprehension*)upb_msg_new(&google_api_expr_v1alpha1_Expr_Comprehension_msginit, arena); + if (!sub) return NULL; + google_api_expr_v1alpha1_Expr_set_comprehension_expr(msg, sub); + } + return sub; +} + +/* google.api.expr.v1alpha1.Expr.Ident */ + +UPB_INLINE google_api_expr_v1alpha1_Expr_Ident *google_api_expr_v1alpha1_Expr_Ident_new(upb_arena *arena) { + return (google_api_expr_v1alpha1_Expr_Ident *)upb_msg_new(&google_api_expr_v1alpha1_Expr_Ident_msginit, arena); +} +UPB_INLINE google_api_expr_v1alpha1_Expr_Ident *google_api_expr_v1alpha1_Expr_Ident_parse(const char *buf, size_t size, + upb_arena *arena) { + google_api_expr_v1alpha1_Expr_Ident *ret = google_api_expr_v1alpha1_Expr_Ident_new(arena); + return (ret && upb_decode(buf, size, ret, &google_api_expr_v1alpha1_Expr_Ident_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_api_expr_v1alpha1_Expr_Ident_serialize(const google_api_expr_v1alpha1_Expr_Ident *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_api_expr_v1alpha1_Expr_Ident_msginit, arena, len); +} + +UPB_INLINE upb_strview google_api_expr_v1alpha1_Expr_Ident_name(const google_api_expr_v1alpha1_Expr_Ident *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(0, 0)); } + +UPB_INLINE void google_api_expr_v1alpha1_Expr_Ident_set_name(google_api_expr_v1alpha1_Expr_Ident *msg, upb_strview value) { + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(0, 0)) = value; +} + +/* google.api.expr.v1alpha1.Expr.Select */ + +UPB_INLINE google_api_expr_v1alpha1_Expr_Select *google_api_expr_v1alpha1_Expr_Select_new(upb_arena *arena) { + return (google_api_expr_v1alpha1_Expr_Select *)upb_msg_new(&google_api_expr_v1alpha1_Expr_Select_msginit, arena); +} +UPB_INLINE google_api_expr_v1alpha1_Expr_Select *google_api_expr_v1alpha1_Expr_Select_parse(const char *buf, size_t size, + upb_arena *arena) { + google_api_expr_v1alpha1_Expr_Select *ret = google_api_expr_v1alpha1_Expr_Select_new(arena); + return (ret && upb_decode(buf, size, ret, &google_api_expr_v1alpha1_Expr_Select_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_api_expr_v1alpha1_Expr_Select_serialize(const google_api_expr_v1alpha1_Expr_Select *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_api_expr_v1alpha1_Expr_Select_msginit, arena, len); +} + +UPB_INLINE const google_api_expr_v1alpha1_Expr* google_api_expr_v1alpha1_Expr_Select_operand(const google_api_expr_v1alpha1_Expr_Select *msg) { return UPB_FIELD_AT(msg, const google_api_expr_v1alpha1_Expr*, UPB_SIZE(12, 24)); } +UPB_INLINE upb_strview google_api_expr_v1alpha1_Expr_Select_field(const google_api_expr_v1alpha1_Expr_Select *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(4, 8)); } +UPB_INLINE bool google_api_expr_v1alpha1_Expr_Select_test_only(const google_api_expr_v1alpha1_Expr_Select *msg) { return UPB_FIELD_AT(msg, bool, UPB_SIZE(0, 0)); } + +UPB_INLINE void google_api_expr_v1alpha1_Expr_Select_set_operand(google_api_expr_v1alpha1_Expr_Select *msg, google_api_expr_v1alpha1_Expr* value) { + UPB_FIELD_AT(msg, google_api_expr_v1alpha1_Expr*, UPB_SIZE(12, 24)) = value; +} +UPB_INLINE struct google_api_expr_v1alpha1_Expr* google_api_expr_v1alpha1_Expr_Select_mutable_operand(google_api_expr_v1alpha1_Expr_Select *msg, upb_arena *arena) { + struct google_api_expr_v1alpha1_Expr* sub = (struct google_api_expr_v1alpha1_Expr*)google_api_expr_v1alpha1_Expr_Select_operand(msg); + if (sub == NULL) { + sub = (struct google_api_expr_v1alpha1_Expr*)upb_msg_new(&google_api_expr_v1alpha1_Expr_msginit, arena); + if (!sub) return NULL; + google_api_expr_v1alpha1_Expr_Select_set_operand(msg, sub); + } + return sub; +} +UPB_INLINE void google_api_expr_v1alpha1_Expr_Select_set_field(google_api_expr_v1alpha1_Expr_Select *msg, upb_strview value) { + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(4, 8)) = value; +} +UPB_INLINE void google_api_expr_v1alpha1_Expr_Select_set_test_only(google_api_expr_v1alpha1_Expr_Select *msg, bool value) { + UPB_FIELD_AT(msg, bool, UPB_SIZE(0, 0)) = value; +} + +/* google.api.expr.v1alpha1.Expr.Call */ + +UPB_INLINE google_api_expr_v1alpha1_Expr_Call *google_api_expr_v1alpha1_Expr_Call_new(upb_arena *arena) { + return (google_api_expr_v1alpha1_Expr_Call *)upb_msg_new(&google_api_expr_v1alpha1_Expr_Call_msginit, arena); +} +UPB_INLINE google_api_expr_v1alpha1_Expr_Call *google_api_expr_v1alpha1_Expr_Call_parse(const char *buf, size_t size, + upb_arena *arena) { + google_api_expr_v1alpha1_Expr_Call *ret = google_api_expr_v1alpha1_Expr_Call_new(arena); + return (ret && upb_decode(buf, size, ret, &google_api_expr_v1alpha1_Expr_Call_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_api_expr_v1alpha1_Expr_Call_serialize(const google_api_expr_v1alpha1_Expr_Call *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_api_expr_v1alpha1_Expr_Call_msginit, arena, len); +} + +UPB_INLINE const google_api_expr_v1alpha1_Expr* google_api_expr_v1alpha1_Expr_Call_target(const google_api_expr_v1alpha1_Expr_Call *msg) { return UPB_FIELD_AT(msg, const google_api_expr_v1alpha1_Expr*, UPB_SIZE(8, 16)); } +UPB_INLINE upb_strview google_api_expr_v1alpha1_Expr_Call_function(const google_api_expr_v1alpha1_Expr_Call *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(0, 0)); } +UPB_INLINE const google_api_expr_v1alpha1_Expr* const* google_api_expr_v1alpha1_Expr_Call_args(const google_api_expr_v1alpha1_Expr_Call *msg, size_t *len) { return (const google_api_expr_v1alpha1_Expr* const*)_upb_array_accessor(msg, UPB_SIZE(12, 24), len); } + +UPB_INLINE void google_api_expr_v1alpha1_Expr_Call_set_target(google_api_expr_v1alpha1_Expr_Call *msg, google_api_expr_v1alpha1_Expr* value) { + UPB_FIELD_AT(msg, google_api_expr_v1alpha1_Expr*, UPB_SIZE(8, 16)) = value; +} +UPB_INLINE struct google_api_expr_v1alpha1_Expr* google_api_expr_v1alpha1_Expr_Call_mutable_target(google_api_expr_v1alpha1_Expr_Call *msg, upb_arena *arena) { + struct google_api_expr_v1alpha1_Expr* sub = (struct google_api_expr_v1alpha1_Expr*)google_api_expr_v1alpha1_Expr_Call_target(msg); + if (sub == NULL) { + sub = (struct google_api_expr_v1alpha1_Expr*)upb_msg_new(&google_api_expr_v1alpha1_Expr_msginit, arena); + if (!sub) return NULL; + google_api_expr_v1alpha1_Expr_Call_set_target(msg, sub); + } + return sub; +} +UPB_INLINE void google_api_expr_v1alpha1_Expr_Call_set_function(google_api_expr_v1alpha1_Expr_Call *msg, upb_strview value) { + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(0, 0)) = value; +} +UPB_INLINE google_api_expr_v1alpha1_Expr** google_api_expr_v1alpha1_Expr_Call_mutable_args(google_api_expr_v1alpha1_Expr_Call *msg, size_t *len) { + return (google_api_expr_v1alpha1_Expr**)_upb_array_mutable_accessor(msg, UPB_SIZE(12, 24), len); +} +UPB_INLINE google_api_expr_v1alpha1_Expr** google_api_expr_v1alpha1_Expr_Call_resize_args(google_api_expr_v1alpha1_Expr_Call *msg, size_t len, upb_arena *arena) { + return (google_api_expr_v1alpha1_Expr**)_upb_array_resize_accessor(msg, UPB_SIZE(12, 24), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_api_expr_v1alpha1_Expr* google_api_expr_v1alpha1_Expr_Call_add_args(google_api_expr_v1alpha1_Expr_Call *msg, upb_arena *arena) { + struct google_api_expr_v1alpha1_Expr* sub = (struct google_api_expr_v1alpha1_Expr*)upb_msg_new(&google_api_expr_v1alpha1_Expr_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(12, 24), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} + +/* google.api.expr.v1alpha1.Expr.CreateList */ + +UPB_INLINE google_api_expr_v1alpha1_Expr_CreateList *google_api_expr_v1alpha1_Expr_CreateList_new(upb_arena *arena) { + return (google_api_expr_v1alpha1_Expr_CreateList *)upb_msg_new(&google_api_expr_v1alpha1_Expr_CreateList_msginit, arena); +} +UPB_INLINE google_api_expr_v1alpha1_Expr_CreateList *google_api_expr_v1alpha1_Expr_CreateList_parse(const char *buf, size_t size, + upb_arena *arena) { + google_api_expr_v1alpha1_Expr_CreateList *ret = google_api_expr_v1alpha1_Expr_CreateList_new(arena); + return (ret && upb_decode(buf, size, ret, &google_api_expr_v1alpha1_Expr_CreateList_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_api_expr_v1alpha1_Expr_CreateList_serialize(const google_api_expr_v1alpha1_Expr_CreateList *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_api_expr_v1alpha1_Expr_CreateList_msginit, arena, len); +} + +UPB_INLINE const google_api_expr_v1alpha1_Expr* const* google_api_expr_v1alpha1_Expr_CreateList_elements(const google_api_expr_v1alpha1_Expr_CreateList *msg, size_t *len) { return (const google_api_expr_v1alpha1_Expr* const*)_upb_array_accessor(msg, UPB_SIZE(0, 0), len); } + +UPB_INLINE google_api_expr_v1alpha1_Expr** google_api_expr_v1alpha1_Expr_CreateList_mutable_elements(google_api_expr_v1alpha1_Expr_CreateList *msg, size_t *len) { + return (google_api_expr_v1alpha1_Expr**)_upb_array_mutable_accessor(msg, UPB_SIZE(0, 0), len); +} +UPB_INLINE google_api_expr_v1alpha1_Expr** google_api_expr_v1alpha1_Expr_CreateList_resize_elements(google_api_expr_v1alpha1_Expr_CreateList *msg, size_t len, upb_arena *arena) { + return (google_api_expr_v1alpha1_Expr**)_upb_array_resize_accessor(msg, UPB_SIZE(0, 0), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_api_expr_v1alpha1_Expr* google_api_expr_v1alpha1_Expr_CreateList_add_elements(google_api_expr_v1alpha1_Expr_CreateList *msg, upb_arena *arena) { + struct google_api_expr_v1alpha1_Expr* sub = (struct google_api_expr_v1alpha1_Expr*)upb_msg_new(&google_api_expr_v1alpha1_Expr_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(0, 0), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} + +/* google.api.expr.v1alpha1.Expr.CreateStruct */ + +UPB_INLINE google_api_expr_v1alpha1_Expr_CreateStruct *google_api_expr_v1alpha1_Expr_CreateStruct_new(upb_arena *arena) { + return (google_api_expr_v1alpha1_Expr_CreateStruct *)upb_msg_new(&google_api_expr_v1alpha1_Expr_CreateStruct_msginit, arena); +} +UPB_INLINE google_api_expr_v1alpha1_Expr_CreateStruct *google_api_expr_v1alpha1_Expr_CreateStruct_parse(const char *buf, size_t size, + upb_arena *arena) { + google_api_expr_v1alpha1_Expr_CreateStruct *ret = google_api_expr_v1alpha1_Expr_CreateStruct_new(arena); + return (ret && upb_decode(buf, size, ret, &google_api_expr_v1alpha1_Expr_CreateStruct_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_api_expr_v1alpha1_Expr_CreateStruct_serialize(const google_api_expr_v1alpha1_Expr_CreateStruct *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_api_expr_v1alpha1_Expr_CreateStruct_msginit, arena, len); +} + +UPB_INLINE upb_strview google_api_expr_v1alpha1_Expr_CreateStruct_message_name(const google_api_expr_v1alpha1_Expr_CreateStruct *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(0, 0)); } +UPB_INLINE const google_api_expr_v1alpha1_Expr_CreateStruct_Entry* const* google_api_expr_v1alpha1_Expr_CreateStruct_entries(const google_api_expr_v1alpha1_Expr_CreateStruct *msg, size_t *len) { return (const google_api_expr_v1alpha1_Expr_CreateStruct_Entry* const*)_upb_array_accessor(msg, UPB_SIZE(8, 16), len); } + +UPB_INLINE void google_api_expr_v1alpha1_Expr_CreateStruct_set_message_name(google_api_expr_v1alpha1_Expr_CreateStruct *msg, upb_strview value) { + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(0, 0)) = value; +} +UPB_INLINE google_api_expr_v1alpha1_Expr_CreateStruct_Entry** google_api_expr_v1alpha1_Expr_CreateStruct_mutable_entries(google_api_expr_v1alpha1_Expr_CreateStruct *msg, size_t *len) { + return (google_api_expr_v1alpha1_Expr_CreateStruct_Entry**)_upb_array_mutable_accessor(msg, UPB_SIZE(8, 16), len); +} +UPB_INLINE google_api_expr_v1alpha1_Expr_CreateStruct_Entry** google_api_expr_v1alpha1_Expr_CreateStruct_resize_entries(google_api_expr_v1alpha1_Expr_CreateStruct *msg, size_t len, upb_arena *arena) { + return (google_api_expr_v1alpha1_Expr_CreateStruct_Entry**)_upb_array_resize_accessor(msg, UPB_SIZE(8, 16), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_api_expr_v1alpha1_Expr_CreateStruct_Entry* google_api_expr_v1alpha1_Expr_CreateStruct_add_entries(google_api_expr_v1alpha1_Expr_CreateStruct *msg, upb_arena *arena) { + struct google_api_expr_v1alpha1_Expr_CreateStruct_Entry* sub = (struct google_api_expr_v1alpha1_Expr_CreateStruct_Entry*)upb_msg_new(&google_api_expr_v1alpha1_Expr_CreateStruct_Entry_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(8, 16), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} + +/* google.api.expr.v1alpha1.Expr.CreateStruct.Entry */ + +UPB_INLINE google_api_expr_v1alpha1_Expr_CreateStruct_Entry *google_api_expr_v1alpha1_Expr_CreateStruct_Entry_new(upb_arena *arena) { + return (google_api_expr_v1alpha1_Expr_CreateStruct_Entry *)upb_msg_new(&google_api_expr_v1alpha1_Expr_CreateStruct_Entry_msginit, arena); +} +UPB_INLINE google_api_expr_v1alpha1_Expr_CreateStruct_Entry *google_api_expr_v1alpha1_Expr_CreateStruct_Entry_parse(const char *buf, size_t size, + upb_arena *arena) { + google_api_expr_v1alpha1_Expr_CreateStruct_Entry *ret = google_api_expr_v1alpha1_Expr_CreateStruct_Entry_new(arena); + return (ret && upb_decode(buf, size, ret, &google_api_expr_v1alpha1_Expr_CreateStruct_Entry_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_api_expr_v1alpha1_Expr_CreateStruct_Entry_serialize(const google_api_expr_v1alpha1_Expr_CreateStruct_Entry *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_api_expr_v1alpha1_Expr_CreateStruct_Entry_msginit, arena, len); +} + +typedef enum { + google_api_expr_v1alpha1_Expr_CreateStruct_Entry_key_kind_field_key = 2, + google_api_expr_v1alpha1_Expr_CreateStruct_Entry_key_kind_map_key = 3, + google_api_expr_v1alpha1_Expr_CreateStruct_Entry_key_kind_NOT_SET = 0 +} google_api_expr_v1alpha1_Expr_CreateStruct_Entry_key_kind_oneofcases; +UPB_INLINE google_api_expr_v1alpha1_Expr_CreateStruct_Entry_key_kind_oneofcases google_api_expr_v1alpha1_Expr_CreateStruct_Entry_key_kind_case(const google_api_expr_v1alpha1_Expr_CreateStruct_Entry* msg) { return (google_api_expr_v1alpha1_Expr_CreateStruct_Entry_key_kind_oneofcases)UPB_FIELD_AT(msg, int32_t, UPB_SIZE(20, 32)); } + +UPB_INLINE int64_t google_api_expr_v1alpha1_Expr_CreateStruct_Entry_id(const google_api_expr_v1alpha1_Expr_CreateStruct_Entry *msg) { return UPB_FIELD_AT(msg, int64_t, UPB_SIZE(0, 0)); } +UPB_INLINE bool google_api_expr_v1alpha1_Expr_CreateStruct_Entry_has_field_key(const google_api_expr_v1alpha1_Expr_CreateStruct_Entry *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(20, 32), 2); } +UPB_INLINE upb_strview google_api_expr_v1alpha1_Expr_CreateStruct_Entry_field_key(const google_api_expr_v1alpha1_Expr_CreateStruct_Entry *msg) { return UPB_READ_ONEOF(msg, upb_strview, UPB_SIZE(12, 16), UPB_SIZE(20, 32), 2, upb_strview_make("", strlen(""))); } +UPB_INLINE bool google_api_expr_v1alpha1_Expr_CreateStruct_Entry_has_map_key(const google_api_expr_v1alpha1_Expr_CreateStruct_Entry *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(20, 32), 3); } +UPB_INLINE const google_api_expr_v1alpha1_Expr* google_api_expr_v1alpha1_Expr_CreateStruct_Entry_map_key(const google_api_expr_v1alpha1_Expr_CreateStruct_Entry *msg) { return UPB_READ_ONEOF(msg, const google_api_expr_v1alpha1_Expr*, UPB_SIZE(12, 16), UPB_SIZE(20, 32), 3, NULL); } +UPB_INLINE const google_api_expr_v1alpha1_Expr* google_api_expr_v1alpha1_Expr_CreateStruct_Entry_value(const google_api_expr_v1alpha1_Expr_CreateStruct_Entry *msg) { return UPB_FIELD_AT(msg, const google_api_expr_v1alpha1_Expr*, UPB_SIZE(8, 8)); } + +UPB_INLINE void google_api_expr_v1alpha1_Expr_CreateStruct_Entry_set_id(google_api_expr_v1alpha1_Expr_CreateStruct_Entry *msg, int64_t value) { + UPB_FIELD_AT(msg, int64_t, UPB_SIZE(0, 0)) = value; +} +UPB_INLINE void google_api_expr_v1alpha1_Expr_CreateStruct_Entry_set_field_key(google_api_expr_v1alpha1_Expr_CreateStruct_Entry *msg, upb_strview value) { + UPB_WRITE_ONEOF(msg, upb_strview, UPB_SIZE(12, 16), value, UPB_SIZE(20, 32), 2); +} +UPB_INLINE void google_api_expr_v1alpha1_Expr_CreateStruct_Entry_set_map_key(google_api_expr_v1alpha1_Expr_CreateStruct_Entry *msg, google_api_expr_v1alpha1_Expr* value) { + UPB_WRITE_ONEOF(msg, google_api_expr_v1alpha1_Expr*, UPB_SIZE(12, 16), value, UPB_SIZE(20, 32), 3); +} +UPB_INLINE struct google_api_expr_v1alpha1_Expr* google_api_expr_v1alpha1_Expr_CreateStruct_Entry_mutable_map_key(google_api_expr_v1alpha1_Expr_CreateStruct_Entry *msg, upb_arena *arena) { + struct google_api_expr_v1alpha1_Expr* sub = (struct google_api_expr_v1alpha1_Expr*)google_api_expr_v1alpha1_Expr_CreateStruct_Entry_map_key(msg); + if (sub == NULL) { + sub = (struct google_api_expr_v1alpha1_Expr*)upb_msg_new(&google_api_expr_v1alpha1_Expr_msginit, arena); + if (!sub) return NULL; + google_api_expr_v1alpha1_Expr_CreateStruct_Entry_set_map_key(msg, sub); + } + return sub; +} +UPB_INLINE void google_api_expr_v1alpha1_Expr_CreateStruct_Entry_set_value(google_api_expr_v1alpha1_Expr_CreateStruct_Entry *msg, google_api_expr_v1alpha1_Expr* value) { + UPB_FIELD_AT(msg, google_api_expr_v1alpha1_Expr*, UPB_SIZE(8, 8)) = value; +} +UPB_INLINE struct google_api_expr_v1alpha1_Expr* google_api_expr_v1alpha1_Expr_CreateStruct_Entry_mutable_value(google_api_expr_v1alpha1_Expr_CreateStruct_Entry *msg, upb_arena *arena) { + struct google_api_expr_v1alpha1_Expr* sub = (struct google_api_expr_v1alpha1_Expr*)google_api_expr_v1alpha1_Expr_CreateStruct_Entry_value(msg); + if (sub == NULL) { + sub = (struct google_api_expr_v1alpha1_Expr*)upb_msg_new(&google_api_expr_v1alpha1_Expr_msginit, arena); + if (!sub) return NULL; + google_api_expr_v1alpha1_Expr_CreateStruct_Entry_set_value(msg, sub); + } + return sub; +} + +/* google.api.expr.v1alpha1.Expr.Comprehension */ + +UPB_INLINE google_api_expr_v1alpha1_Expr_Comprehension *google_api_expr_v1alpha1_Expr_Comprehension_new(upb_arena *arena) { + return (google_api_expr_v1alpha1_Expr_Comprehension *)upb_msg_new(&google_api_expr_v1alpha1_Expr_Comprehension_msginit, arena); +} +UPB_INLINE google_api_expr_v1alpha1_Expr_Comprehension *google_api_expr_v1alpha1_Expr_Comprehension_parse(const char *buf, size_t size, + upb_arena *arena) { + google_api_expr_v1alpha1_Expr_Comprehension *ret = google_api_expr_v1alpha1_Expr_Comprehension_new(arena); + return (ret && upb_decode(buf, size, ret, &google_api_expr_v1alpha1_Expr_Comprehension_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_api_expr_v1alpha1_Expr_Comprehension_serialize(const google_api_expr_v1alpha1_Expr_Comprehension *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_api_expr_v1alpha1_Expr_Comprehension_msginit, arena, len); +} + +UPB_INLINE upb_strview google_api_expr_v1alpha1_Expr_Comprehension_iter_var(const google_api_expr_v1alpha1_Expr_Comprehension *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(0, 0)); } +UPB_INLINE const google_api_expr_v1alpha1_Expr* google_api_expr_v1alpha1_Expr_Comprehension_iter_range(const google_api_expr_v1alpha1_Expr_Comprehension *msg) { return UPB_FIELD_AT(msg, const google_api_expr_v1alpha1_Expr*, UPB_SIZE(16, 32)); } +UPB_INLINE upb_strview google_api_expr_v1alpha1_Expr_Comprehension_accu_var(const google_api_expr_v1alpha1_Expr_Comprehension *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(8, 16)); } +UPB_INLINE const google_api_expr_v1alpha1_Expr* google_api_expr_v1alpha1_Expr_Comprehension_accu_init(const google_api_expr_v1alpha1_Expr_Comprehension *msg) { return UPB_FIELD_AT(msg, const google_api_expr_v1alpha1_Expr*, UPB_SIZE(20, 40)); } +UPB_INLINE const google_api_expr_v1alpha1_Expr* google_api_expr_v1alpha1_Expr_Comprehension_loop_condition(const google_api_expr_v1alpha1_Expr_Comprehension *msg) { return UPB_FIELD_AT(msg, const google_api_expr_v1alpha1_Expr*, UPB_SIZE(24, 48)); } +UPB_INLINE const google_api_expr_v1alpha1_Expr* google_api_expr_v1alpha1_Expr_Comprehension_loop_step(const google_api_expr_v1alpha1_Expr_Comprehension *msg) { return UPB_FIELD_AT(msg, const google_api_expr_v1alpha1_Expr*, UPB_SIZE(28, 56)); } +UPB_INLINE const google_api_expr_v1alpha1_Expr* google_api_expr_v1alpha1_Expr_Comprehension_result(const google_api_expr_v1alpha1_Expr_Comprehension *msg) { return UPB_FIELD_AT(msg, const google_api_expr_v1alpha1_Expr*, UPB_SIZE(32, 64)); } + +UPB_INLINE void google_api_expr_v1alpha1_Expr_Comprehension_set_iter_var(google_api_expr_v1alpha1_Expr_Comprehension *msg, upb_strview value) { + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(0, 0)) = value; +} +UPB_INLINE void google_api_expr_v1alpha1_Expr_Comprehension_set_iter_range(google_api_expr_v1alpha1_Expr_Comprehension *msg, google_api_expr_v1alpha1_Expr* value) { + UPB_FIELD_AT(msg, google_api_expr_v1alpha1_Expr*, UPB_SIZE(16, 32)) = value; +} +UPB_INLINE struct google_api_expr_v1alpha1_Expr* google_api_expr_v1alpha1_Expr_Comprehension_mutable_iter_range(google_api_expr_v1alpha1_Expr_Comprehension *msg, upb_arena *arena) { + struct google_api_expr_v1alpha1_Expr* sub = (struct google_api_expr_v1alpha1_Expr*)google_api_expr_v1alpha1_Expr_Comprehension_iter_range(msg); + if (sub == NULL) { + sub = (struct google_api_expr_v1alpha1_Expr*)upb_msg_new(&google_api_expr_v1alpha1_Expr_msginit, arena); + if (!sub) return NULL; + google_api_expr_v1alpha1_Expr_Comprehension_set_iter_range(msg, sub); + } + return sub; +} +UPB_INLINE void google_api_expr_v1alpha1_Expr_Comprehension_set_accu_var(google_api_expr_v1alpha1_Expr_Comprehension *msg, upb_strview value) { + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(8, 16)) = value; +} +UPB_INLINE void google_api_expr_v1alpha1_Expr_Comprehension_set_accu_init(google_api_expr_v1alpha1_Expr_Comprehension *msg, google_api_expr_v1alpha1_Expr* value) { + UPB_FIELD_AT(msg, google_api_expr_v1alpha1_Expr*, UPB_SIZE(20, 40)) = value; +} +UPB_INLINE struct google_api_expr_v1alpha1_Expr* google_api_expr_v1alpha1_Expr_Comprehension_mutable_accu_init(google_api_expr_v1alpha1_Expr_Comprehension *msg, upb_arena *arena) { + struct google_api_expr_v1alpha1_Expr* sub = (struct google_api_expr_v1alpha1_Expr*)google_api_expr_v1alpha1_Expr_Comprehension_accu_init(msg); + if (sub == NULL) { + sub = (struct google_api_expr_v1alpha1_Expr*)upb_msg_new(&google_api_expr_v1alpha1_Expr_msginit, arena); + if (!sub) return NULL; + google_api_expr_v1alpha1_Expr_Comprehension_set_accu_init(msg, sub); + } + return sub; +} +UPB_INLINE void google_api_expr_v1alpha1_Expr_Comprehension_set_loop_condition(google_api_expr_v1alpha1_Expr_Comprehension *msg, google_api_expr_v1alpha1_Expr* value) { + UPB_FIELD_AT(msg, google_api_expr_v1alpha1_Expr*, UPB_SIZE(24, 48)) = value; +} +UPB_INLINE struct google_api_expr_v1alpha1_Expr* google_api_expr_v1alpha1_Expr_Comprehension_mutable_loop_condition(google_api_expr_v1alpha1_Expr_Comprehension *msg, upb_arena *arena) { + struct google_api_expr_v1alpha1_Expr* sub = (struct google_api_expr_v1alpha1_Expr*)google_api_expr_v1alpha1_Expr_Comprehension_loop_condition(msg); + if (sub == NULL) { + sub = (struct google_api_expr_v1alpha1_Expr*)upb_msg_new(&google_api_expr_v1alpha1_Expr_msginit, arena); + if (!sub) return NULL; + google_api_expr_v1alpha1_Expr_Comprehension_set_loop_condition(msg, sub); + } + return sub; +} +UPB_INLINE void google_api_expr_v1alpha1_Expr_Comprehension_set_loop_step(google_api_expr_v1alpha1_Expr_Comprehension *msg, google_api_expr_v1alpha1_Expr* value) { + UPB_FIELD_AT(msg, google_api_expr_v1alpha1_Expr*, UPB_SIZE(28, 56)) = value; +} +UPB_INLINE struct google_api_expr_v1alpha1_Expr* google_api_expr_v1alpha1_Expr_Comprehension_mutable_loop_step(google_api_expr_v1alpha1_Expr_Comprehension *msg, upb_arena *arena) { + struct google_api_expr_v1alpha1_Expr* sub = (struct google_api_expr_v1alpha1_Expr*)google_api_expr_v1alpha1_Expr_Comprehension_loop_step(msg); + if (sub == NULL) { + sub = (struct google_api_expr_v1alpha1_Expr*)upb_msg_new(&google_api_expr_v1alpha1_Expr_msginit, arena); + if (!sub) return NULL; + google_api_expr_v1alpha1_Expr_Comprehension_set_loop_step(msg, sub); + } + return sub; +} +UPB_INLINE void google_api_expr_v1alpha1_Expr_Comprehension_set_result(google_api_expr_v1alpha1_Expr_Comprehension *msg, google_api_expr_v1alpha1_Expr* value) { + UPB_FIELD_AT(msg, google_api_expr_v1alpha1_Expr*, UPB_SIZE(32, 64)) = value; +} +UPB_INLINE struct google_api_expr_v1alpha1_Expr* google_api_expr_v1alpha1_Expr_Comprehension_mutable_result(google_api_expr_v1alpha1_Expr_Comprehension *msg, upb_arena *arena) { + struct google_api_expr_v1alpha1_Expr* sub = (struct google_api_expr_v1alpha1_Expr*)google_api_expr_v1alpha1_Expr_Comprehension_result(msg); + if (sub == NULL) { + sub = (struct google_api_expr_v1alpha1_Expr*)upb_msg_new(&google_api_expr_v1alpha1_Expr_msginit, arena); + if (!sub) return NULL; + google_api_expr_v1alpha1_Expr_Comprehension_set_result(msg, sub); + } + return sub; +} + +/* google.api.expr.v1alpha1.Constant */ + +UPB_INLINE google_api_expr_v1alpha1_Constant *google_api_expr_v1alpha1_Constant_new(upb_arena *arena) { + return (google_api_expr_v1alpha1_Constant *)upb_msg_new(&google_api_expr_v1alpha1_Constant_msginit, arena); +} +UPB_INLINE google_api_expr_v1alpha1_Constant *google_api_expr_v1alpha1_Constant_parse(const char *buf, size_t size, + upb_arena *arena) { + google_api_expr_v1alpha1_Constant *ret = google_api_expr_v1alpha1_Constant_new(arena); + return (ret && upb_decode(buf, size, ret, &google_api_expr_v1alpha1_Constant_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_api_expr_v1alpha1_Constant_serialize(const google_api_expr_v1alpha1_Constant *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_api_expr_v1alpha1_Constant_msginit, arena, len); +} + +typedef enum { + google_api_expr_v1alpha1_Constant_constant_kind_null_value = 1, + google_api_expr_v1alpha1_Constant_constant_kind_bool_value = 2, + google_api_expr_v1alpha1_Constant_constant_kind_int64_value = 3, + google_api_expr_v1alpha1_Constant_constant_kind_uint64_value = 4, + google_api_expr_v1alpha1_Constant_constant_kind_double_value = 5, + google_api_expr_v1alpha1_Constant_constant_kind_string_value = 6, + google_api_expr_v1alpha1_Constant_constant_kind_bytes_value = 7, + google_api_expr_v1alpha1_Constant_constant_kind_duration_value = 8, + google_api_expr_v1alpha1_Constant_constant_kind_timestamp_value = 9, + google_api_expr_v1alpha1_Constant_constant_kind_NOT_SET = 0 +} google_api_expr_v1alpha1_Constant_constant_kind_oneofcases; +UPB_INLINE google_api_expr_v1alpha1_Constant_constant_kind_oneofcases google_api_expr_v1alpha1_Constant_constant_kind_case(const google_api_expr_v1alpha1_Constant* msg) { return (google_api_expr_v1alpha1_Constant_constant_kind_oneofcases)UPB_FIELD_AT(msg, int32_t, UPB_SIZE(8, 16)); } + +UPB_INLINE bool google_api_expr_v1alpha1_Constant_has_null_value(const google_api_expr_v1alpha1_Constant *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(8, 16), 1); } +UPB_INLINE int32_t google_api_expr_v1alpha1_Constant_null_value(const google_api_expr_v1alpha1_Constant *msg) { return UPB_READ_ONEOF(msg, int32_t, UPB_SIZE(0, 0), UPB_SIZE(8, 16), 1, google_protobuf_NULL_VALUE); } +UPB_INLINE bool google_api_expr_v1alpha1_Constant_has_bool_value(const google_api_expr_v1alpha1_Constant *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(8, 16), 2); } +UPB_INLINE bool google_api_expr_v1alpha1_Constant_bool_value(const google_api_expr_v1alpha1_Constant *msg) { return UPB_READ_ONEOF(msg, bool, UPB_SIZE(0, 0), UPB_SIZE(8, 16), 2, false); } +UPB_INLINE bool google_api_expr_v1alpha1_Constant_has_int64_value(const google_api_expr_v1alpha1_Constant *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(8, 16), 3); } +UPB_INLINE int64_t google_api_expr_v1alpha1_Constant_int64_value(const google_api_expr_v1alpha1_Constant *msg) { return UPB_READ_ONEOF(msg, int64_t, UPB_SIZE(0, 0), UPB_SIZE(8, 16), 3, 0); } +UPB_INLINE bool google_api_expr_v1alpha1_Constant_has_uint64_value(const google_api_expr_v1alpha1_Constant *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(8, 16), 4); } +UPB_INLINE uint64_t google_api_expr_v1alpha1_Constant_uint64_value(const google_api_expr_v1alpha1_Constant *msg) { return UPB_READ_ONEOF(msg, uint64_t, UPB_SIZE(0, 0), UPB_SIZE(8, 16), 4, 0); } +UPB_INLINE bool google_api_expr_v1alpha1_Constant_has_double_value(const google_api_expr_v1alpha1_Constant *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(8, 16), 5); } +UPB_INLINE double google_api_expr_v1alpha1_Constant_double_value(const google_api_expr_v1alpha1_Constant *msg) { return UPB_READ_ONEOF(msg, double, UPB_SIZE(0, 0), UPB_SIZE(8, 16), 5, 0); } +UPB_INLINE bool google_api_expr_v1alpha1_Constant_has_string_value(const google_api_expr_v1alpha1_Constant *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(8, 16), 6); } +UPB_INLINE upb_strview google_api_expr_v1alpha1_Constant_string_value(const google_api_expr_v1alpha1_Constant *msg) { return UPB_READ_ONEOF(msg, upb_strview, UPB_SIZE(0, 0), UPB_SIZE(8, 16), 6, upb_strview_make("", strlen(""))); } +UPB_INLINE bool google_api_expr_v1alpha1_Constant_has_bytes_value(const google_api_expr_v1alpha1_Constant *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(8, 16), 7); } +UPB_INLINE upb_strview google_api_expr_v1alpha1_Constant_bytes_value(const google_api_expr_v1alpha1_Constant *msg) { return UPB_READ_ONEOF(msg, upb_strview, UPB_SIZE(0, 0), UPB_SIZE(8, 16), 7, upb_strview_make("", strlen(""))); } +UPB_INLINE bool google_api_expr_v1alpha1_Constant_has_duration_value(const google_api_expr_v1alpha1_Constant *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(8, 16), 8); } +UPB_INLINE const struct google_protobuf_Duration* google_api_expr_v1alpha1_Constant_duration_value(const google_api_expr_v1alpha1_Constant *msg) { return UPB_READ_ONEOF(msg, const struct google_protobuf_Duration*, UPB_SIZE(0, 0), UPB_SIZE(8, 16), 8, NULL); } +UPB_INLINE bool google_api_expr_v1alpha1_Constant_has_timestamp_value(const google_api_expr_v1alpha1_Constant *msg) { return _upb_has_oneof_field(msg, UPB_SIZE(8, 16), 9); } +UPB_INLINE const struct google_protobuf_Timestamp* google_api_expr_v1alpha1_Constant_timestamp_value(const google_api_expr_v1alpha1_Constant *msg) { return UPB_READ_ONEOF(msg, const struct google_protobuf_Timestamp*, UPB_SIZE(0, 0), UPB_SIZE(8, 16), 9, NULL); } + +UPB_INLINE void google_api_expr_v1alpha1_Constant_set_null_value(google_api_expr_v1alpha1_Constant *msg, int32_t value) { + UPB_WRITE_ONEOF(msg, int32_t, UPB_SIZE(0, 0), value, UPB_SIZE(8, 16), 1); +} +UPB_INLINE void google_api_expr_v1alpha1_Constant_set_bool_value(google_api_expr_v1alpha1_Constant *msg, bool value) { + UPB_WRITE_ONEOF(msg, bool, UPB_SIZE(0, 0), value, UPB_SIZE(8, 16), 2); +} +UPB_INLINE void google_api_expr_v1alpha1_Constant_set_int64_value(google_api_expr_v1alpha1_Constant *msg, int64_t value) { + UPB_WRITE_ONEOF(msg, int64_t, UPB_SIZE(0, 0), value, UPB_SIZE(8, 16), 3); +} +UPB_INLINE void google_api_expr_v1alpha1_Constant_set_uint64_value(google_api_expr_v1alpha1_Constant *msg, uint64_t value) { + UPB_WRITE_ONEOF(msg, uint64_t, UPB_SIZE(0, 0), value, UPB_SIZE(8, 16), 4); +} +UPB_INLINE void google_api_expr_v1alpha1_Constant_set_double_value(google_api_expr_v1alpha1_Constant *msg, double value) { + UPB_WRITE_ONEOF(msg, double, UPB_SIZE(0, 0), value, UPB_SIZE(8, 16), 5); +} +UPB_INLINE void google_api_expr_v1alpha1_Constant_set_string_value(google_api_expr_v1alpha1_Constant *msg, upb_strview value) { + UPB_WRITE_ONEOF(msg, upb_strview, UPB_SIZE(0, 0), value, UPB_SIZE(8, 16), 6); +} +UPB_INLINE void google_api_expr_v1alpha1_Constant_set_bytes_value(google_api_expr_v1alpha1_Constant *msg, upb_strview value) { + UPB_WRITE_ONEOF(msg, upb_strview, UPB_SIZE(0, 0), value, UPB_SIZE(8, 16), 7); +} +UPB_INLINE void google_api_expr_v1alpha1_Constant_set_duration_value(google_api_expr_v1alpha1_Constant *msg, struct google_protobuf_Duration* value) { + UPB_WRITE_ONEOF(msg, struct google_protobuf_Duration*, UPB_SIZE(0, 0), value, UPB_SIZE(8, 16), 8); +} +UPB_INLINE struct google_protobuf_Duration* google_api_expr_v1alpha1_Constant_mutable_duration_value(google_api_expr_v1alpha1_Constant *msg, upb_arena *arena) { + struct google_protobuf_Duration* sub = (struct google_protobuf_Duration*)google_api_expr_v1alpha1_Constant_duration_value(msg); + if (sub == NULL) { + sub = (struct google_protobuf_Duration*)upb_msg_new(&google_protobuf_Duration_msginit, arena); + if (!sub) return NULL; + google_api_expr_v1alpha1_Constant_set_duration_value(msg, sub); + } + return sub; +} +UPB_INLINE void google_api_expr_v1alpha1_Constant_set_timestamp_value(google_api_expr_v1alpha1_Constant *msg, struct google_protobuf_Timestamp* value) { + UPB_WRITE_ONEOF(msg, struct google_protobuf_Timestamp*, UPB_SIZE(0, 0), value, UPB_SIZE(8, 16), 9); +} +UPB_INLINE struct google_protobuf_Timestamp* google_api_expr_v1alpha1_Constant_mutable_timestamp_value(google_api_expr_v1alpha1_Constant *msg, upb_arena *arena) { + struct google_protobuf_Timestamp* sub = (struct google_protobuf_Timestamp*)google_api_expr_v1alpha1_Constant_timestamp_value(msg); + if (sub == NULL) { + sub = (struct google_protobuf_Timestamp*)upb_msg_new(&google_protobuf_Timestamp_msginit, arena); + if (!sub) return NULL; + google_api_expr_v1alpha1_Constant_set_timestamp_value(msg, sub); + } + return sub; +} + +/* google.api.expr.v1alpha1.SourceInfo */ + +UPB_INLINE google_api_expr_v1alpha1_SourceInfo *google_api_expr_v1alpha1_SourceInfo_new(upb_arena *arena) { + return (google_api_expr_v1alpha1_SourceInfo *)upb_msg_new(&google_api_expr_v1alpha1_SourceInfo_msginit, arena); +} +UPB_INLINE google_api_expr_v1alpha1_SourceInfo *google_api_expr_v1alpha1_SourceInfo_parse(const char *buf, size_t size, + upb_arena *arena) { + google_api_expr_v1alpha1_SourceInfo *ret = google_api_expr_v1alpha1_SourceInfo_new(arena); + return (ret && upb_decode(buf, size, ret, &google_api_expr_v1alpha1_SourceInfo_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_api_expr_v1alpha1_SourceInfo_serialize(const google_api_expr_v1alpha1_SourceInfo *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_api_expr_v1alpha1_SourceInfo_msginit, arena, len); +} + +UPB_INLINE upb_strview google_api_expr_v1alpha1_SourceInfo_syntax_version(const google_api_expr_v1alpha1_SourceInfo *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(0, 0)); } +UPB_INLINE upb_strview google_api_expr_v1alpha1_SourceInfo_location(const google_api_expr_v1alpha1_SourceInfo *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(8, 16)); } +UPB_INLINE int32_t const* google_api_expr_v1alpha1_SourceInfo_line_offsets(const google_api_expr_v1alpha1_SourceInfo *msg, size_t *len) { return (int32_t const*)_upb_array_accessor(msg, UPB_SIZE(16, 32), len); } +UPB_INLINE const google_api_expr_v1alpha1_SourceInfo_PositionsEntry* const* google_api_expr_v1alpha1_SourceInfo_positions(const google_api_expr_v1alpha1_SourceInfo *msg, size_t *len) { return (const google_api_expr_v1alpha1_SourceInfo_PositionsEntry* const*)_upb_array_accessor(msg, UPB_SIZE(20, 40), len); } + +UPB_INLINE void google_api_expr_v1alpha1_SourceInfo_set_syntax_version(google_api_expr_v1alpha1_SourceInfo *msg, upb_strview value) { + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(0, 0)) = value; +} +UPB_INLINE void google_api_expr_v1alpha1_SourceInfo_set_location(google_api_expr_v1alpha1_SourceInfo *msg, upb_strview value) { + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(8, 16)) = value; +} +UPB_INLINE int32_t* google_api_expr_v1alpha1_SourceInfo_mutable_line_offsets(google_api_expr_v1alpha1_SourceInfo *msg, size_t *len) { + return (int32_t*)_upb_array_mutable_accessor(msg, UPB_SIZE(16, 32), len); +} +UPB_INLINE int32_t* google_api_expr_v1alpha1_SourceInfo_resize_line_offsets(google_api_expr_v1alpha1_SourceInfo *msg, size_t len, upb_arena *arena) { + return (int32_t*)_upb_array_resize_accessor(msg, UPB_SIZE(16, 32), len, UPB_SIZE(4, 4), UPB_TYPE_INT32, arena); +} +UPB_INLINE bool google_api_expr_v1alpha1_SourceInfo_add_line_offsets(google_api_expr_v1alpha1_SourceInfo *msg, int32_t val, upb_arena *arena) { + return _upb_array_append_accessor( + msg, UPB_SIZE(16, 32), UPB_SIZE(4, 4), UPB_TYPE_INT32, &val, arena); +} +UPB_INLINE google_api_expr_v1alpha1_SourceInfo_PositionsEntry** google_api_expr_v1alpha1_SourceInfo_mutable_positions(google_api_expr_v1alpha1_SourceInfo *msg, size_t *len) { + return (google_api_expr_v1alpha1_SourceInfo_PositionsEntry**)_upb_array_mutable_accessor(msg, UPB_SIZE(20, 40), len); +} +UPB_INLINE google_api_expr_v1alpha1_SourceInfo_PositionsEntry** google_api_expr_v1alpha1_SourceInfo_resize_positions(google_api_expr_v1alpha1_SourceInfo *msg, size_t len, upb_arena *arena) { + return (google_api_expr_v1alpha1_SourceInfo_PositionsEntry**)_upb_array_resize_accessor(msg, UPB_SIZE(20, 40), len, UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, arena); +} +UPB_INLINE struct google_api_expr_v1alpha1_SourceInfo_PositionsEntry* google_api_expr_v1alpha1_SourceInfo_add_positions(google_api_expr_v1alpha1_SourceInfo *msg, upb_arena *arena) { + struct google_api_expr_v1alpha1_SourceInfo_PositionsEntry* sub = (struct google_api_expr_v1alpha1_SourceInfo_PositionsEntry*)upb_msg_new(&google_api_expr_v1alpha1_SourceInfo_PositionsEntry_msginit, arena); + bool ok = _upb_array_append_accessor( + msg, UPB_SIZE(20, 40), UPB_SIZE(4, 8), UPB_TYPE_MESSAGE, &sub, arena); + if (!ok) return NULL; + return sub; +} + +/* google.api.expr.v1alpha1.SourceInfo.PositionsEntry */ + +UPB_INLINE google_api_expr_v1alpha1_SourceInfo_PositionsEntry *google_api_expr_v1alpha1_SourceInfo_PositionsEntry_new(upb_arena *arena) { + return (google_api_expr_v1alpha1_SourceInfo_PositionsEntry *)upb_msg_new(&google_api_expr_v1alpha1_SourceInfo_PositionsEntry_msginit, arena); +} +UPB_INLINE google_api_expr_v1alpha1_SourceInfo_PositionsEntry *google_api_expr_v1alpha1_SourceInfo_PositionsEntry_parse(const char *buf, size_t size, + upb_arena *arena) { + google_api_expr_v1alpha1_SourceInfo_PositionsEntry *ret = google_api_expr_v1alpha1_SourceInfo_PositionsEntry_new(arena); + return (ret && upb_decode(buf, size, ret, &google_api_expr_v1alpha1_SourceInfo_PositionsEntry_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_api_expr_v1alpha1_SourceInfo_PositionsEntry_serialize(const google_api_expr_v1alpha1_SourceInfo_PositionsEntry *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_api_expr_v1alpha1_SourceInfo_PositionsEntry_msginit, arena, len); +} + +UPB_INLINE int64_t google_api_expr_v1alpha1_SourceInfo_PositionsEntry_key(const google_api_expr_v1alpha1_SourceInfo_PositionsEntry *msg) { return UPB_FIELD_AT(msg, int64_t, UPB_SIZE(0, 0)); } +UPB_INLINE int32_t google_api_expr_v1alpha1_SourceInfo_PositionsEntry_value(const google_api_expr_v1alpha1_SourceInfo_PositionsEntry *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(8, 8)); } + +UPB_INLINE void google_api_expr_v1alpha1_SourceInfo_PositionsEntry_set_key(google_api_expr_v1alpha1_SourceInfo_PositionsEntry *msg, int64_t value) { + UPB_FIELD_AT(msg, int64_t, UPB_SIZE(0, 0)) = value; +} +UPB_INLINE void google_api_expr_v1alpha1_SourceInfo_PositionsEntry_set_value(google_api_expr_v1alpha1_SourceInfo_PositionsEntry *msg, int32_t value) { + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(8, 8)) = value; +} + +/* google.api.expr.v1alpha1.SourcePosition */ + +UPB_INLINE google_api_expr_v1alpha1_SourcePosition *google_api_expr_v1alpha1_SourcePosition_new(upb_arena *arena) { + return (google_api_expr_v1alpha1_SourcePosition *)upb_msg_new(&google_api_expr_v1alpha1_SourcePosition_msginit, arena); +} +UPB_INLINE google_api_expr_v1alpha1_SourcePosition *google_api_expr_v1alpha1_SourcePosition_parse(const char *buf, size_t size, + upb_arena *arena) { + google_api_expr_v1alpha1_SourcePosition *ret = google_api_expr_v1alpha1_SourcePosition_new(arena); + return (ret && upb_decode(buf, size, ret, &google_api_expr_v1alpha1_SourcePosition_msginit, arena)) ? ret : NULL; +} +UPB_INLINE char *google_api_expr_v1alpha1_SourcePosition_serialize(const google_api_expr_v1alpha1_SourcePosition *msg, upb_arena *arena, size_t *len) { + return upb_encode(msg, &google_api_expr_v1alpha1_SourcePosition_msginit, arena, len); +} + +UPB_INLINE upb_strview google_api_expr_v1alpha1_SourcePosition_location(const google_api_expr_v1alpha1_SourcePosition *msg) { return UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(12, 16)); } +UPB_INLINE int32_t google_api_expr_v1alpha1_SourcePosition_offset(const google_api_expr_v1alpha1_SourcePosition *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(0, 0)); } +UPB_INLINE int32_t google_api_expr_v1alpha1_SourcePosition_line(const google_api_expr_v1alpha1_SourcePosition *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(4, 4)); } +UPB_INLINE int32_t google_api_expr_v1alpha1_SourcePosition_column(const google_api_expr_v1alpha1_SourcePosition *msg) { return UPB_FIELD_AT(msg, int32_t, UPB_SIZE(8, 8)); } + +UPB_INLINE void google_api_expr_v1alpha1_SourcePosition_set_location(google_api_expr_v1alpha1_SourcePosition *msg, upb_strview value) { + UPB_FIELD_AT(msg, upb_strview, UPB_SIZE(12, 16)) = value; +} +UPB_INLINE void google_api_expr_v1alpha1_SourcePosition_set_offset(google_api_expr_v1alpha1_SourcePosition *msg, int32_t value) { + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(0, 0)) = value; +} +UPB_INLINE void google_api_expr_v1alpha1_SourcePosition_set_line(google_api_expr_v1alpha1_SourcePosition *msg, int32_t value) { + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(4, 4)) = value; +} +UPB_INLINE void google_api_expr_v1alpha1_SourcePosition_set_column(google_api_expr_v1alpha1_SourcePosition *msg, int32_t value) { + UPB_FIELD_AT(msg, int32_t, UPB_SIZE(8, 8)) = value; +} + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#include "upb/port_undef.inc" + +#endif /* GOOGLE_API_EXPR_V1ALPHA1_SYNTAX_PROTO_UPB_H_ */ diff --git a/tools/codegen/core/gen_upb_api.sh b/tools/codegen/core/gen_upb_api.sh index 51d29930142..97541d24a89 100755 --- a/tools/codegen/core/gen_upb_api.sh +++ b/tools/codegen/core/gen_upb_api.sh @@ -74,12 +74,14 @@ proto_files=( \ "envoy/config/listener/v2/api_listener.proto" \ "envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto" \ "envoy/config/filter/accesslog/v2/accesslog.proto" \ + "envoy/config/rbac/v2/rbac.proto" \ "envoy/config/trace/v2/http_tracer.proto" \ "envoy/service/discovery/v2/ads.proto" \ "envoy/service/load_stats/v2/lrs.proto" \ "envoy/type/http.proto" \ "envoy/type/matcher/regex.proto" \ "envoy/api/v2/listener/udp_listener_config.proto" \ + "envoy/type/matcher/path.proto" \ "envoy/type/matcher/string.proto" \ "envoy/type/metadata/v2/metadata.proto" \ "envoy/type/percent.proto" \ @@ -88,6 +90,7 @@ proto_files=( \ "envoy/type/tracing/v2/custom_tag.proto" \ "gogoproto/gogo.proto" \ "google/api/annotations.proto" \ + "google/api/expr/v1alpha1/syntax.proto" \ "google/api/http.proto" \ "google/protobuf/any.proto" \ "google/protobuf/descriptor.proto" \ From e0ca3b1939d431607ba0b045b32828186d8ef1d4 Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Fri, 12 Jun 2020 10:01:33 -0700 Subject: [PATCH 28/80] Pin version of RSA --- requirements.bazel.txt | 1 + requirements.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/requirements.bazel.txt b/requirements.bazel.txt index 4d52beb9e11..cffce19e778 100644 --- a/requirements.bazel.txt +++ b/requirements.bazel.txt @@ -14,3 +14,4 @@ chardet==3.0.4 certifi==2017.4.17 idna==2.7 googleapis-common-protos==1.5.5 +rsa>=3.1.4,!=4.1,!=4.1.1,!=4.2,<5 diff --git a/requirements.txt b/requirements.txt index 27dd7d9f63b..799e75bca41 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ enum34>=1.0.4 protobuf>=3.5.0.post1 six>=1.10 wheel>=0.29 +rsa>=3.1.4,!=4.1,!=4.1.1,!=4.2,<5 From 3bef88ca7e21454cbea05ad312c284a6f608202b Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Fri, 12 Jun 2020 10:09:16 -0700 Subject: [PATCH 29/80] Pin to 4.0 --- requirements.bazel.txt | 3 ++- requirements.txt | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/requirements.bazel.txt b/requirements.bazel.txt index cffce19e778..586c710f108 100644 --- a/requirements.bazel.txt +++ b/requirements.bazel.txt @@ -14,4 +14,5 @@ chardet==3.0.4 certifi==2017.4.17 idna==2.7 googleapis-common-protos==1.5.5 -rsa>=3.1.4,!=4.1,!=4.1.1,!=4.2,<5 +# rsa 4.0 is the last version support Python 2 +rsa==4.0 diff --git a/requirements.txt b/requirements.txt index 799e75bca41..2218991c3f3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,5 @@ enum34>=1.0.4 protobuf>=3.5.0.post1 six>=1.10 wheel>=0.29 -rsa>=3.1.4,!=4.1,!=4.1.1,!=4.2,<5 +# rsa 4.0 is the last version support Python 2 +rsa==4.0 From 9a3ae476586fbf3ca21600f3e4711fecd1f0aebd Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Fri, 12 Jun 2020 10:48:47 -0700 Subject: [PATCH 30/80] Pump up the minimum version of google-auth to 1.17.1 --- requirements.bazel.txt | 2 +- src/python/grpcio_tests/setup.py | 4 ++-- tools/run_tests/helper_scripts/build_python.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.bazel.txt b/requirements.bazel.txt index 586c710f108..d103d3004c5 100644 --- a/requirements.bazel.txt +++ b/requirements.bazel.txt @@ -6,7 +6,7 @@ protobuf>=3.5.0.post1 six>=1.10 wheel>=0.29 futures>=2.2.0 -google-auth>=1.0.0 +google-auth>=1.17.1 oauth2client==4.1.0 requests>=2.14.2 urllib3>=1.23 diff --git a/src/python/grpcio_tests/setup.py b/src/python/grpcio_tests/setup.py index 671f0653732..67d22fd7148 100644 --- a/src/python/grpcio_tests/setup.py +++ b/src/python/grpcio_tests/setup.py @@ -43,8 +43,8 @@ INSTALL_REQUIRES = ( 'grpcio-status>={version}'.format(version=grpc_version.VERSION), 'grpcio-tools>={version}'.format(version=grpc_version.VERSION), 'grpcio-health-checking>={version}'.format(version=grpc_version.VERSION), - 'oauth2client>=1.4.7', 'protobuf>=3.6.0', 'six>=1.10', 'google-auth>=1.0.0', - 'requests>=2.14.2') + 'oauth2client>=1.4.7', 'protobuf>=3.6.0', 'six>=1.10', + 'google-auth>=1.17.1', 'requests>=2.14.2') if not PY3: INSTALL_REQUIRES += ('futures>=2.2.0',) diff --git a/tools/run_tests/helper_scripts/build_python.sh b/tools/run_tests/helper_scripts/build_python.sh index e79a8896092..47e96fdb19f 100755 --- a/tools/run_tests/helper_scripts/build_python.sh +++ b/tools/run_tests/helper_scripts/build_python.sh @@ -224,7 +224,7 @@ pip_install_dir "$ROOT/src/python/grpcio_testing" # Build/install tests pip_install coverage==4.4 oauth2client==4.1.0 \ - google-auth==1.0.0 requests==2.14.2 \ + google-auth==1.17.1 requests==2.14.2 \ googleapis-common-protos==1.5.5 $VENV_PYTHON "$ROOT/src/python/grpcio_tests/setup.py" preprocess $VENV_PYTHON "$ROOT/src/python/grpcio_tests/setup.py" build_package_protos From 25491305ff8826853cc3256f53b263c0504af578 Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Fri, 12 Jun 2020 10:57:57 -0700 Subject: [PATCH 31/80] Pin to 4.3 instead of 4.0 --- requirements.bazel.txt | 4 ++-- requirements.txt | 4 ++-- tools/run_tests/helper_scripts/build_python.sh | 3 +++ 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/requirements.bazel.txt b/requirements.bazel.txt index d103d3004c5..0e2ecbe0907 100644 --- a/requirements.bazel.txt +++ b/requirements.bazel.txt @@ -14,5 +14,5 @@ chardet==3.0.4 certifi==2017.4.17 idna==2.7 googleapis-common-protos==1.5.5 -# rsa 4.0 is the last version support Python 2 -rsa==4.0 +# rsa 4.3 is the last version support Python 2 +rsa==4.3 diff --git a/requirements.txt b/requirements.txt index 2218991c3f3..773e2c7c79b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,5 +5,5 @@ enum34>=1.0.4 protobuf>=3.5.0.post1 six>=1.10 wheel>=0.29 -# rsa 4.0 is the last version support Python 2 -rsa==4.0 +# rsa 4.3 is the last version support Python 2 +rsa==4.3 diff --git a/tools/run_tests/helper_scripts/build_python.sh b/tools/run_tests/helper_scripts/build_python.sh index 47e96fdb19f..0b0cf380a1e 100755 --- a/tools/run_tests/helper_scripts/build_python.sh +++ b/tools/run_tests/helper_scripts/build_python.sh @@ -222,6 +222,9 @@ pip_install_dir "$ROOT/src/python/grpcio_status" # Install testing pip_install_dir "$ROOT/src/python/grpcio_testing" +# rsa 4.3 is the latest version support Python 2 +pip_install ras==4.3 + # Build/install tests pip_install coverage==4.4 oauth2client==4.1.0 \ google-auth==1.17.1 requests==2.14.2 \ From 3f9e2148ebfdcb60a86dad93ed94c332d3d97cd6 Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Fri, 12 Jun 2020 11:14:44 -0700 Subject: [PATCH 32/80] Fix typo ras -> rsa --- tools/run_tests/helper_scripts/build_python.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/helper_scripts/build_python.sh b/tools/run_tests/helper_scripts/build_python.sh index 0b0cf380a1e..699aa849972 100755 --- a/tools/run_tests/helper_scripts/build_python.sh +++ b/tools/run_tests/helper_scripts/build_python.sh @@ -223,7 +223,7 @@ pip_install_dir "$ROOT/src/python/grpcio_status" pip_install_dir "$ROOT/src/python/grpcio_testing" # rsa 4.3 is the latest version support Python 2 -pip_install ras==4.3 +pip_install rsa==4.3 # Build/install tests pip_install coverage==4.4 oauth2client==4.1.0 \ From 5e6b089f16570e3e4c2dcc7626f4c8c56ddf1e11 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Fri, 12 Jun 2020 12:18:57 -0700 Subject: [PATCH 33/80] Print outstanding allocations in log --- src/core/lib/iomgr/resource_quota.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/lib/iomgr/resource_quota.cc b/src/core/lib/iomgr/resource_quota.cc index 967d0b7e4f4..de5b25a7827 100644 --- a/src/core/lib/iomgr/resource_quota.cc +++ b/src/core/lib/iomgr/resource_quota.cc @@ -320,9 +320,9 @@ static bool rq_alloc(grpc_resource_quota* resource_quota) { if (GRPC_TRACE_FLAG_ENABLED(grpc_resource_quota_trace)) { gpr_log(GPR_INFO, "RQ: check allocation for user %p shutdown=%" PRIdPTR - " free_pool=%" PRId64, + " free_pool=%" PRId64 " outstanding_allocations=%" PRId64, resource_user, gpr_atm_no_barrier_load(&resource_user->shutdown), - resource_user->free_pool); + resource_user->free_pool, resource_user->outstanding_allocations); } if (gpr_atm_no_barrier_load(&resource_user->shutdown)) { resource_user->allocating = false; From 72f2b99097b6ec36cb8a91cb09ed161f8fe48dfd Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Fri, 12 Jun 2020 13:07:23 -0700 Subject: [PATCH 34/80] Guard _ChannelCallState.__del__ from cygrpc being deallocated early --- src/python/grpcio/grpc/_channel.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/python/grpcio/grpc/_channel.py b/src/python/grpcio/grpc/_channel.py index 8bae8a12127..3e8a6b75a9d 100644 --- a/src/python/grpcio/grpc/_channel.py +++ b/src/python/grpcio/grpc/_channel.py @@ -1123,7 +1123,8 @@ class _ChannelCallState(object): self.managed_calls = 0 def __del__(self): - if hasattr(self, 'channel') and self.channel: + if hasattr(self, + 'channel') and self.channel and cygrpc and cygrpc.StatusCode: self.channel.close(cygrpc.StatusCode.cancelled, 'Channel deallocated!') From 0eefd8b20981d90ca453bfa4794b253ba6364654 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Fri, 12 Jun 2020 14:22:34 -0700 Subject: [PATCH 35/80] Finish client first completely for client-lead tests --- test/cpp/qps/driver.cc | 42 ++++++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/test/cpp/qps/driver.cc b/test/cpp/qps/driver.cc index deac6c43cbe..59640ebb5cc 100644 --- a/test/cpp/qps/driver.cc +++ b/test/cpp/qps/driver.cc @@ -492,6 +492,10 @@ std::unique_ptr RunScenario( Histogram merged_latencies; std::unordered_map merged_statuses; + // For the case where clients leads the test, it's going to finish + // clients first and wait until it's completely done for generaous termination. + bool client_finish_first = (client_config.rpc_type() != STREAMING_FROM_SERVER); + gpr_log(GPR_INFO, "Finishing clients"); for (size_t i = 0; i < num_clients; i++) { auto client = &clients[i]; @@ -504,16 +508,19 @@ std::unique_ptr RunScenario( GPR_ASSERT(false); } } - gpr_log(GPR_INFO, "Finishing servers"); - for (size_t i = 0; i < num_servers; i++) { - auto server = &servers[i]; - if (!server->stream->Write(server_mark)) { - gpr_log(GPR_ERROR, "Couldn't write mark to server %zu", i); - GPR_ASSERT(false); - } - if (!server->stream->WritesDone()) { - gpr_log(GPR_ERROR, "Failed WritesDone for server %zu", i); - GPR_ASSERT(false); + + if (!client_finish_first) { + gpr_log(GPR_INFO, "Finishing servers"); + for (size_t i = 0; i < num_servers; i++) { + auto server = &servers[i]; + if (!server->stream->Write(server_mark)) { + gpr_log(GPR_ERROR, "Couldn't write mark to server %zu", i); + GPR_ASSERT(false); + } + if (!server->stream->WritesDone()) { + gpr_log(GPR_ERROR, "Failed WritesDone for server %zu", i); + GPR_ASSERT(false); + } } } @@ -559,6 +566,21 @@ std::unique_ptr RunScenario( rrc->set_count(it->second); } + if (client_finish_first) { + gpr_log(GPR_INFO, "Finishing servers"); + for (size_t i = 0; i < num_servers; i++) { + auto server = &servers[i]; + if (!server->stream->Write(server_mark)) { + gpr_log(GPR_ERROR, "Couldn't write mark to server %zu", i); + GPR_ASSERT(false); + } + if (!server->stream->WritesDone()) { + gpr_log(GPR_ERROR, "Failed WritesDone for server %zu", i); + GPR_ASSERT(false); + } + } + } + for (size_t i = 0; i < num_servers; i++) { auto server = &servers[i]; // Read the server final status From b35b463a08382869c7bb4a28c6839b526963add1 Mon Sep 17 00:00:00 2001 From: Donna Dionne Date: Fri, 12 Jun 2020 15:19:32 -0700 Subject: [PATCH 36/80] reject RouteConfiguration with a case-insensitive RouteMatch --- .../ext/filters/client_channel/xds/xds_api.cc | 7 +++ .../grpc/testing/xds/lds_rds_for_test.proto | 5 +++ test/cpp/end2end/xds_end2end_test.cc | 43 +++++++++++++++++++ 3 files changed, 55 insertions(+) diff --git a/src/core/ext/filters/client_channel/xds/xds_api.cc b/src/core/ext/filters/client_channel/xds/xds_api.cc index 69490b776fd..d4c2f380f50 100644 --- a/src/core/ext/filters/client_channel/xds/xds_api.cc +++ b/src/core/ext/filters/client_channel/xds/xds_api.cc @@ -1081,6 +1081,13 @@ grpc_error* RouteConfigParse( const envoy_api_v2_route_Route* route = routes[i]; const envoy_api_v2_route_RouteMatch* match = envoy_api_v2_route_Route_match(route); + const google_protobuf_BoolValue* case_sensitive = + envoy_api_v2_route_RouteMatch_case_sensitive(match); + if (case_sensitive != nullptr && + !google_protobuf_BoolValue_value(case_sensitive)) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "case_sensitive if set must be set to true."); + } XdsApi::RdsUpdate::RdsRoute rds_route; if (envoy_api_v2_route_RouteMatch_has_prefix(match)) { upb_strview prefix = envoy_api_v2_route_RouteMatch_prefix(match); diff --git a/src/proto/grpc/testing/xds/lds_rds_for_test.proto b/src/proto/grpc/testing/xds/lds_rds_for_test.proto index ec2a46b7c88..a4d48b28d1a 100644 --- a/src/proto/grpc/testing/xds/lds_rds_for_test.proto +++ b/src/proto/grpc/testing/xds/lds_rds_for_test.proto @@ -29,6 +29,10 @@ import "google/protobuf/any.proto"; import "google/protobuf/wrappers.proto"; import "src/proto/grpc/testing/xds/cds_for_test.proto"; +message BoolValue { + // The bool value. + bool value = 1; +} message RouteMatch { oneof path_specifier { @@ -37,6 +41,7 @@ message RouteMatch { string prefix = 1; string path = 2; } + BoolValue case_sensitive = 4; } message WeightedCluster { diff --git a/test/cpp/end2end/xds_end2end_test.cc b/test/cpp/end2end/xds_end2end_test.cc index a163c49c24a..688d6cfd987 100644 --- a/test/cpp/end2end/xds_end2end_test.cc +++ b/test/cpp/end2end/xds_end2end_test.cc @@ -2004,6 +2004,26 @@ TEST_P(XdsResolverOnlyTest, DefaultRouteSpecifiesSlashPrefix) { WaitForAllBackends(); } +TEST_P(XdsResolverOnlyTest, DefaultRouteCaseInsensitive) { + RouteConfiguration route_config = + balancers_[0]->ads_service()->default_route_config(); + route_config.mutable_virtual_hosts(0) + ->mutable_routes(0) + ->mutable_match() + ->mutable_case_sensitive() + ->set_value(false); + balancers_[0]->ads_service()->SetLdsResource( + AdsServiceImpl::BuildListener(route_config)); + SetNextResolution({}); + SetNextResolutionForLbChannelAllBalancers(); + AdsServiceImpl::EdsResourceArgs args({ + {"locality0", GetBackendPorts()}, + }); + balancers_[0]->ads_service()->SetEdsResource( + AdsServiceImpl::BuildEdsResource(args)); + CheckRpcSendOk(5, RpcOptions().set_wait_for_ready(true)); +} + class XdsResolverLoadReportingOnlyTest : public XdsEnd2endTest { public: XdsResolverLoadReportingOnlyTest() : XdsEnd2endTest(4, 1, 3) {} @@ -2390,6 +2410,29 @@ TEST_P(LdsRdsTest, RouteMatchHasUnsupportedSpecifier) { "No prefix field found in Default RouteMatch."); } +// Tests that LDS client should send a NACK if route match has a case_sensitive +// set to false. +TEST_P(LdsRdsTest, RouteMatchHasCaseSensitiveFalse) { + gpr_setenv("GRPC_XDS_EXPERIMENTAL_ROUTING", "true"); + RouteConfiguration route_config = + balancers_[0]->ads_service()->default_route_config(); + auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0); + route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/"); + route1->mutable_match()->mutable_case_sensitive()->set_value(false); + auto* default_route = route_config.mutable_virtual_hosts(0)->add_routes(); + default_route->mutable_match()->set_prefix(""); + default_route->mutable_route()->set_cluster(kDefaultResourceName); + SetRouteConfiguration(0, route_config); + SetNextResolution({}); + SetNextResolutionForLbChannelAllBalancers(); + CheckRpcSendFailure(); + const auto& response_state = RouteConfigurationResponseState(0); + EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); + EXPECT_EQ(response_state.error_message, + "case_sensitive if set must be set to true."); + gpr_unsetenv("GRPC_XDS_EXPERIMENTAL_ROUTING"); +} + // Tests that LDS client should send a NACK if route match has a prefix // string with no "/". TEST_P(LdsRdsTest, RouteMatchHasInvalidPrefixNonEmptyNoSlash) { From 5a2f60d137e6a550527c83a7aeef919ecb41a4e2 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Fri, 12 Jun 2020 15:29:49 -0700 Subject: [PATCH 37/80] Allow skipping canceled check --- test/cpp/end2end/flaky_network_test.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/cpp/end2end/flaky_network_test.cc b/test/cpp/end2end/flaky_network_test.cc index ecf715d288f..93c41e55fa9 100644 --- a/test/cpp/end2end/flaky_network_test.cc +++ b/test/cpp/end2end/flaky_network_test.cc @@ -213,6 +213,9 @@ class FlakyNetworkTest : public ::testing::TestWithParam { ClientContext context; if (timeout_ms > 0) { context.set_deadline(grpc_timeout_milliseconds_to_deadline(timeout_ms)); + // Allow an RPC to be canceled (for deadline exceeded) after it has + // reached the server. + request.mutable_param()->set_skip_cancelled_check(true); } // See https://github.com/grpc/grpc/blob/master/doc/wait-for-ready.md for // details of wait-for-ready semantics From ed2037d343147f7c11224b42dde89e42b359cdf6 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Fri, 12 Jun 2020 16:32:02 -0700 Subject: [PATCH 38/80] Update by review --- test/cpp/qps/driver.cc | 260 +++++++++++++++++++++++------------------ 1 file changed, 146 insertions(+), 114 deletions(-) diff --git a/test/cpp/qps/driver.cc b/test/cpp/qps/driver.cc index 59640ebb5cc..a4e474cf4bd 100644 --- a/test/cpp/qps/driver.cc +++ b/test/cpp/qps/driver.cc @@ -227,6 +227,132 @@ static void postprocess_scenario_result(ScenarioResult* result) { client_queries_per_cpu_sec); } +struct ClientData { + unique_ptr stub; + unique_ptr> stream; +}; + +struct ServerData { + unique_ptr stub; + unique_ptr> stream; +}; + +static void FinishClients(const std::vector& clients, + const ClientArgs& client_mark) { + gpr_log(GPR_INFO, "Finishing clients"); + for (size_t i = 0, i_end = clients.size(); i < i_end; i++) { + auto client = &clients[i]; + if (!client->stream->Write(client_mark)) { + gpr_log(GPR_ERROR, "Couldn't write mark to client %zu", i); + GPR_ASSERT(false); + } + if (!client->stream->WritesDone()) { + gpr_log(GPR_ERROR, "Failed WritesDone for client %zu", i); + GPR_ASSERT(false); + } + } +} + +static void ReceiveFinalStatusFromClients( + const std::vector& clients, Histogram& merged_latencies, + std::unordered_map& merged_statuses, ScenarioResult& result) { + gpr_log(GPR_INFO, "Receiving final status from clients"); + ClientStatus client_status; + for (size_t i = 0, i_end = clients.size(); i < i_end; i++) { + auto client = &clients[i]; + // Read the client final status + if (client->stream->Read(&client_status)) { + gpr_log(GPR_INFO, "Received final status from client %zu", i); + const auto& stats = client_status.stats(); + merged_latencies.MergeProto(stats.latencies()); + for (int i = 0; i < stats.request_results_size(); i++) { + merged_statuses[stats.request_results(i).status_code()] += + stats.request_results(i).count(); + } + result.add_client_stats()->CopyFrom(stats); + // That final status should be the last message on the client stream + GPR_ASSERT(!client->stream->Read(&client_status)); + } else { + gpr_log(GPR_ERROR, "Couldn't get final status from client %zu", i); + GPR_ASSERT(false); + } + } +} + +static void ShutdownClients(const std::vector& clients, + ScenarioResult& result) { + gpr_log(GPR_INFO, "Shutdown clients"); + for (size_t i = 0, i_end = clients.size(); i < i_end; i++) { + auto client = &clients[i]; + Status s = client->stream->Finish(); + // Since we shutdown servers and clients at the same time, clients can + // observe cancellation. Thus, we consider both OK and CANCELLED as good + // status. + const bool success = IsSuccess(s); + result.add_client_success(success); + if (!success) { + gpr_log(GPR_ERROR, "Client %zu had an error %s", i, + s.error_message().c_str()); + GPR_ASSERT(false); + } + } +} + +static void FinishServers(const std::vector& servers, + const ServerArgs& server_mark) { + gpr_log(GPR_INFO, "Finishing servers"); + for (size_t i = 0, i_end = servers.size(); i < i_end; i++) { + auto server = &servers[i]; + if (!server->stream->Write(server_mark)) { + gpr_log(GPR_ERROR, "Couldn't write mark to server %zu", i); + GPR_ASSERT(false); + } + if (!server->stream->WritesDone()) { + gpr_log(GPR_ERROR, "Failed WritesDone for server %zu", i); + GPR_ASSERT(false); + } + } +} + +static void ReceiveFinalStatusFromServer(const std::vector& servers, + ScenarioResult& result) { + gpr_log(GPR_INFO, "Receiving final status from servers"); + ServerStatus server_status; + for (size_t i = 0, i_end = servers.size(); i < i_end; i++) { + auto server = &servers[i]; + // Read the server final status + if (server->stream->Read(&server_status)) { + gpr_log(GPR_INFO, "Received final status from server %zu", i); + result.add_server_stats()->CopyFrom(server_status.stats()); + result.add_server_cores(server_status.cores()); + // That final status should be the last message on the server stream + GPR_ASSERT(!server->stream->Read(&server_status)); + } else { + gpr_log(GPR_ERROR, "Couldn't get final status from server %zu", i); + GPR_ASSERT(false); + } + } +} + +static void ShutdownServers(const std::vector& servers, + ScenarioResult& result) { + gpr_log(GPR_INFO, "Shutdown servers"); + for (size_t i = 0, i_end = servers.size(); i < i_end; i++) { + auto server = &servers[i]; + Status s = server->stream->Finish(); + // Since we shutdown servers and clients at the same time, servers can + // observe cancellation. Thus, we consider both OK and CANCELLED as good + // status. + const bool success = IsSuccess(s); + result.add_server_success(success); + if (!success) { + gpr_log(GPR_ERROR, "Server %zu had an error %s", i, + s.error_message().c_str()); + GPR_ASSERT(false); + } + } +} + std::vector* g_inproc_servers = nullptr; std::unique_ptr RunScenario( @@ -301,10 +427,6 @@ std::unique_ptr RunScenario( workers.resize(num_clients + num_servers); // Start servers - struct ServerData { - unique_ptr stub; - unique_ptr> stream; - }; std::vector servers(num_servers); std::unordered_map> hosts_cores; ChannelArguments channel_args; @@ -363,10 +485,6 @@ std::unique_ptr RunScenario( // Targets are all set by now result_client_config = client_config; // Start clients - struct ClientData { - unique_ptr stub; - unique_ptr> stream; - }; std::vector clients(num_clients); size_t channels_allocated = 0; for (size_t i = 0; i < num_clients; i++) { @@ -492,70 +610,32 @@ std::unique_ptr RunScenario( Histogram merged_latencies; std::unordered_map merged_statuses; - // For the case where clients leads the test, it's going to finish - // clients first and wait until it's completely done for generaous termination. - bool client_finish_first = (client_config.rpc_type() != STREAMING_FROM_SERVER); + // For the case where clients lead the test such as UNARY and + // STREAMING_FROM_CLIENT, clients need to finish completely while a server + // is running to prevent the clients from being stuck while waiting for + // the result. + bool client_finish_first = + (client_config.rpc_type() != STREAMING_FROM_SERVER); - gpr_log(GPR_INFO, "Finishing clients"); - for (size_t i = 0; i < num_clients; i++) { - auto client = &clients[i]; - if (!client->stream->Write(client_mark)) { - gpr_log(GPR_ERROR, "Couldn't write mark to client %zu", i); - GPR_ASSERT(false); - } - if (!client->stream->WritesDone()) { - gpr_log(GPR_ERROR, "Failed WritesDone for client %zu", i); - GPR_ASSERT(false); - } - } + FinishClients(clients, client_mark); if (!client_finish_first) { - gpr_log(GPR_INFO, "Finishing servers"); - for (size_t i = 0; i < num_servers; i++) { - auto server = &servers[i]; - if (!server->stream->Write(server_mark)) { - gpr_log(GPR_ERROR, "Couldn't write mark to server %zu", i); - GPR_ASSERT(false); - } - if (!server->stream->WritesDone()) { - gpr_log(GPR_ERROR, "Failed WritesDone for server %zu", i); - GPR_ASSERT(false); - } - } + FinishServers(servers, server_mark); } - for (size_t i = 0; i < num_clients; i++) { - auto client = &clients[i]; - // Read the client final status - if (client->stream->Read(&client_status)) { - gpr_log(GPR_INFO, "Received final status from client %zu", i); - const auto& stats = client_status.stats(); - merged_latencies.MergeProto(stats.latencies()); - for (int i = 0; i < stats.request_results_size(); i++) { - merged_statuses[stats.request_results(i).status_code()] += - stats.request_results(i).count(); - } - result->add_client_stats()->CopyFrom(stats); - // That final status should be the last message on the client stream - GPR_ASSERT(!client->stream->Read(&client_status)); - } else { - gpr_log(GPR_ERROR, "Couldn't get final status from client %zu", i); - GPR_ASSERT(false); - } + ReceiveFinalStatusFromClients(clients, merged_latencies, merged_statuses, + *result); + ShutdownClients(clients, *result); + + if (client_finish_first) { + FinishServers(servers, server_mark); } - for (size_t i = 0; i < num_clients; i++) { - auto client = &clients[i]; - Status s = client->stream->Finish(); - // Since we shutdown servers and clients at the same time, clients can - // observe cancellation. Thus, we consider both OK and CANCELLED as good - // status. - const bool success = IsSuccess(s); - result->add_client_success(success); - if (!success) { - gpr_log(GPR_ERROR, "Client %zu had an error %s", i, - s.error_message().c_str()); - GPR_ASSERT(false); - } + + ReceiveFinalStatusFromServer(servers, *result); + ShutdownServers(servers, *result); + + if (g_inproc_servers != nullptr) { + delete g_inproc_servers; } merged_latencies.FillProto(result->mutable_latencies()); @@ -565,54 +645,6 @@ std::unique_ptr RunScenario( rrc->set_status_code(it->first); rrc->set_count(it->second); } - - if (client_finish_first) { - gpr_log(GPR_INFO, "Finishing servers"); - for (size_t i = 0; i < num_servers; i++) { - auto server = &servers[i]; - if (!server->stream->Write(server_mark)) { - gpr_log(GPR_ERROR, "Couldn't write mark to server %zu", i); - GPR_ASSERT(false); - } - if (!server->stream->WritesDone()) { - gpr_log(GPR_ERROR, "Failed WritesDone for server %zu", i); - GPR_ASSERT(false); - } - } - } - - for (size_t i = 0; i < num_servers; i++) { - auto server = &servers[i]; - // Read the server final status - if (server->stream->Read(&server_status)) { - gpr_log(GPR_INFO, "Received final status from server %zu", i); - result->add_server_stats()->CopyFrom(server_status.stats()); - result->add_server_cores(server_status.cores()); - // That final status should be the last message on the server stream - GPR_ASSERT(!server->stream->Read(&server_status)); - } else { - gpr_log(GPR_ERROR, "Couldn't get final status from server %zu", i); - GPR_ASSERT(false); - } - } - for (size_t i = 0; i < num_servers; i++) { - auto server = &servers[i]; - Status s = server->stream->Finish(); - // Since we shutdown servers and clients at the same time, servers can - // observe cancellation. Thus, we consider both OK and CANCELLED as good - // status. - const bool success = IsSuccess(s); - result->add_server_success(success); - if (!success) { - gpr_log(GPR_ERROR, "Server %zu had an error %s", i, - s.error_message().c_str()); - GPR_ASSERT(false); - } - } - - if (g_inproc_servers != nullptr) { - delete g_inproc_servers; - } postprocess_scenario_result(result.get()); return result; } From e17602ab94425abcf2eb9e8654dac401d829b47d Mon Sep 17 00:00:00 2001 From: srini100 Date: Sun, 14 Jun 2020 14:29:09 -0700 Subject: [PATCH 39/80] update boringssl instructions --- third_party/README.md | 40 ++++++++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/third_party/README.md b/third_party/README.md index 58b9a3a11f1..951734af151 100644 --- a/third_party/README.md +++ b/third_party/README.md @@ -37,23 +37,51 @@ Usually the process is Updating some dependencies requires extra care. -### Updating third_party/boringssl-with bazel - -- Update the `third_party/boringssl-with-bazel` submodule to the latest `master-with-bazel` branch - -- Update boringssl dependency in `grpc_deps.bzl` to the same commit +### Updating third_party/boringssl-with-bazel + +- Update the `third_party/boringssl-with-bazel` submodule to the latest [`master-with-bazel`](https://github.com/google/boringssl/tree/master-with-bazel) branch +``` +git submodule update --init # just to start in a clean state +cd third_party/boringssl-with-bazel +git fetch origin # fetch what's new in the boringssl repository +git checkout origin/master-with-bazel # checkout the current state of master-with-bazel branch in the boringssl repo +# Note the latest commit SHA on master-with-bazel-branch +cd ../.. # go back to grpc repo root +git status # will show that there are new commits in third_party/boringssl-with-bazel +git add third_party/boringssl-with-bazel # we actually want to update the changes to the submodule +git commit -m "update submodule boringssl-with-bazel with origin/master-with-bazel" # commit +``` + +- Update boringssl dependency in `bazel/grpc_deps.bzl` to the same commit SHA as master-with-bazel branch + - Update `http_archive(name = "boringssl",` section by updating the sha in `strip_prefix` and `urls` fields. + - Also, set `sha256` field to “” as the existing value is not valid. This will be added later once we know what that value is. - Update `tools/run_tests/sanity/check_submodules.sh` with the same commit +- Commit these changes `git commit -m "update boringssl dependency to master-with-bazel commit SHA"` + - Run `tools/buildgen/generate_projects.sh` to regenerate the generated files + - Because `sha256` in `bazel/grpc_deps.bzl` was left empty, you will get a DEBUG msg like this one: +``` +Rule 'boringssl' indicated that a canonical reproducible form can be obtained by modifying arguments sha256 = "SHA value" +``` + - Commit the regenrated files `git commit -m "regenerate files"` + - Update `bazel/grpc_deps.bzl` with the SHA value shown in the above debug msg. Commit again `git commit -m "Updated sha256"` -- Run `tools/distrib/generate_grpc_shadow_boringssl_symbol_list.sh` +- Run `tools/distrib/generate_boringssl_prefix_header.sh` + - Commit again `commit -m "generate boringsll prefix headers"` - Increment the boringssl podspec version number in `templates/src/objective-c/BoringSSL-GRPC.podspec.template` and `templates/gRPC-Core.podspec.template`. [example](https://github.com/grpc/grpc/pull/21527/commits/9d4411842f02f167209887f1f3d2b9ab5d14931a) + - Commit again `commit -m "Increment podspec version"` - Run `tools/buildgen/generate_projects.sh` (yes, again) + - Commit again `commit -m "Second regeneration"` + +- Create a PR with all the above commits. + +- Run `bazel/update_mirror.sh` to update GCS mirror. ### Updating third_party/protobuf From 4a8171cda56fa8419e4b2898712feaf40dee7bfb Mon Sep 17 00:00:00 2001 From: srini100 Date: Sun, 14 Jun 2020 20:02:36 -0700 Subject: [PATCH 40/80] xds features per release --- doc/grpc_xds_features.md | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 doc/grpc_xds_features.md diff --git a/doc/grpc_xds_features.md b/doc/grpc_xds_features.md new file mode 100644 index 00000000000..7e0921c88c4 --- /dev/null +++ b/doc/grpc_xds_features.md @@ -0,0 +1,40 @@ +# xDS Features in gRPC + +This document lists the [xDS](https://github.com/envoyproxy/data-plane-api/tree/master/envoy/api/v2) +features supported in various gRPC language implementations and versions. + +Note that a gRPC client will simply ignore the configuration of a feature it +does not support. The gRPC client does not generate a log +to indicate that some configuration was ignored. It is impractical to generate +a log and keep it up-to-date because xDS has a large number of APIs that gRPC +does not support and the APIs keep evolving too. We recommend reading the +[first gRFC](https://github.com/grpc/proposal/blob/master/A27-xds-global-load-balancing.md) +on xDS support in gRPC to understand the design philosophy. + +The EDS policy will *not* support +[overprovisioning](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/load_balancing/overprovisioning), +which is different from Envoy. Envoy takes the overprovisioning into +account in both [locality-weighted load balancing](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/load_balancing/locality_weight) +and [priority failover](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/load_balancing/priority), +but gRPC assumes that the xDS server will update it to redirect traffic +when this kind of graceful failover is needed. gRPC will send the +[`envoy.lb.does_not_support_overprovisioning` client +feature](https://github.com/envoyproxy/envoy/pull/10136) to the xDS +server to tell the xDS server that it will not perform graceful failover; +xDS server implementations may use this to decide whether to perform +graceful failover themselves. + +The EDS policy will not support per-endpoint stats; it will report only +per-locality stats. + +An [`lb_endpoint`](https://github.com/envoyproxy/envoy/blob/master/api/envoy/api/v2/endpoint/endpoint_components.proto) +is ignored if the `health_status` is not HEALTHY or UNKNOWN. +The optional `load_balancing_weight` is always ignored. + +Initially, only `google_default` channel creds will be supported +to authenticate with the xDS server. + +Features | gRFCs | [C++, Python,
Ruby, PHP, C#](https://github.com/grpc/grpc/releases) | [Java](https://github.com/grpc/grpc-java/releases) | [Go](https://github.com/grpc/grpc-go/releases) +---------|--------|--------------|------|------ +**xDS Infrastructure:**
LDS->RDS->CDS->EDS flow,
ADS stream,
Client-side Load reporting via [LRS](https://github.com/envoyproxy/data-plane-api/blob/master/envoy/service/load_stats/v2/lrs.proto), | [A27](https://github.com/grpc/proposal/blob/master/A27-xds-global-load-balancing.md) | v1.30.0 | v1.30.0 | v1.30.0 | +**Load Balancing:**
Virtual host matching,
Only default path ("" or "/") matching,
Priority-based weighted round-robin locality picking,
Round-robin endpoint picking within locality,
Cluster route action| [A27](https://github.com/grpc/proposal/blob/master/A27-xds-global-load-balancing.md) | v1.30.0 | v1.30.0 | v1.30.0 | From 92725c384daf8f05be933cdb04225268d4b3b186 Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Mon, 15 Jun 2020 11:08:50 -0700 Subject: [PATCH 41/80] packages.grpc.io: Use search.maven.org instead of mvnrepository.com It is unafficiated with Maven Central. Just link to the normal site. This is approximately the same link the grpc-java README uses. Technically, https://repo.maven.apache.org/maven2/io/grpc/ is potentially even better, since that is by Apache instead of Sonatype and is the _actual_ repository, but the difference probably doesn't matter to most people. --- tools/package_hosting/home.xsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/package_hosting/home.xsl b/tools/package_hosting/home.xsl index 8550953174c..7aa5ccb24e8 100644 --- a/tools/package_hosting/home.xsl +++ b/tools/package_hosting/home.xsl @@ -45,7 +45,7 @@
  • C#: NuGet package Grpc
  • Dart: pub package grpc
  • Go: go get google.golang.org/grpc
    • -
  • Java: Use JARs from gRPC Maven Central Repository
    • +
  • Java: Use JARs from gRPC Maven Central Repository
  • Kotlin: Use JARs from gRPC Maven Central Repository
  • Node: npm install grpc
  • Objective-C: Add gRPC-ProtoRPC dependency to podspec
    • From 3040f92254ffa920d51e86eb36b8ac0efd48cb15 Mon Sep 17 00:00:00 2001 From: srini100 Date: Mon, 15 Jun 2020 11:28:20 -0700 Subject: [PATCH 42/80] update boringssl instructions --- third_party/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/README.md b/third_party/README.md index 951734af151..76f6d67aafb 100644 --- a/third_party/README.md +++ b/third_party/README.md @@ -69,7 +69,7 @@ Rule 'boringssl' indicated that a canonical reproducible form can be obtained by - Update `bazel/grpc_deps.bzl` with the SHA value shown in the above debug msg. Commit again `git commit -m "Updated sha256"` - Run `tools/distrib/generate_boringssl_prefix_header.sh` - - Commit again `commit -m "generate boringsll prefix headers"` + - Commit again `commit -m "generate boringssl prefix headers"` - Increment the boringssl podspec version number in `templates/src/objective-c/BoringSSL-GRPC.podspec.template` and `templates/gRPC-Core.podspec.template`. From 060d61e3fb2d6d41edbb0e8a8398b40bf6dd834f Mon Sep 17 00:00:00 2001 From: yulin-liang Date: Thu, 11 Jun 2020 11:06:00 -0700 Subject: [PATCH 43/80] Cronet transport: do an extra read for a trailer-only stream with grpc_status = 0 to trigger on_succeeded() callback. --- .../cronet/transport/cronet_transport.cc | 43 ++++++++----------- 1 file changed, 19 insertions(+), 24 deletions(-) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.cc b/src/core/ext/transport/cronet/transport/cronet_transport.cc index b3a0c401dbc..fc0357bbc73 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.cc +++ b/src/core/ext/transport/cronet/transport/cronet_transport.cc @@ -317,6 +317,17 @@ static void maybe_flush_read(stream_obj* s) { } } +static void read_grpc_header(stream_obj* s) { + s->state.rs.read_buffer = s->state.rs.grpc_header_bytes; + s->state.rs.remaining_bytes = GRPC_HEADER_SIZE_IN_BYTES; + s->state.rs.received_bytes = 0; + s->state.rs.compressed = false; + CRONET_LOG(GPR_DEBUG, "bidirectional_stream_read(%p)", s->cbs); + bidirectional_stream_read(s->cbs, s->state.rs.read_buffer, + s->state.rs.remaining_bytes); + s->state.pending_read_from_cronet = true; +} + static grpc_error* make_error_with_desc(int error_code, const char* desc) { grpc_error* error = GRPC_ERROR_CREATE_FROM_COPIED_STRING(desc); error = grpc_error_set_int(error, GRPC_ERROR_INT_GRPC_STATUS, error_code); @@ -555,6 +566,11 @@ static void on_response_headers_received( for (size_t i = 0; i < headers->count; i++) { if (0 == strcmp("grpc-status", headers->headers[i].key)) { on_response_trailers_received(stream, headers); + /* Do an extra read for a trailer-only stream with grpc_status = 0 + to trigger on_succeeded() callback */ + if (0 == strcmp(headers->headers[i].value, "0")) { + read_grpc_header(s); + } return; } } @@ -567,14 +583,7 @@ static void on_response_headers_received( /* Do an extra read to trigger on_succeeded() callback in case connection is closed */ GPR_ASSERT(s->state.rs.length_field_received == false); - s->state.rs.read_buffer = s->state.rs.grpc_header_bytes; - s->state.rs.compressed = false; - s->state.rs.received_bytes = 0; - s->state.rs.remaining_bytes = GRPC_HEADER_SIZE_IN_BYTES; - CRONET_LOG(GPR_DEBUG, "bidirectional_stream_read(%p)", s->cbs); - bidirectional_stream_read(s->cbs, s->state.rs.read_buffer, - s->state.rs.remaining_bytes); - s->state.pending_read_from_cronet = true; + read_grpc_header(s); } gpr_mu_unlock(&s->mu); execute_from_storage(s); @@ -1260,17 +1269,10 @@ static enum e_op_result execute_stream_op(struct op_and_state* oas) { oas->state.state_op_done[OP_RECV_MESSAGE] = true; /* Extra read to trigger on_succeed */ - stream_state->rs.read_buffer = stream_state->rs.grpc_header_bytes; - stream_state->rs.remaining_bytes = GRPC_HEADER_SIZE_IN_BYTES; - stream_state->rs.received_bytes = 0; - stream_state->rs.compressed = false; stream_state->rs.length_field_received = false; - CRONET_LOG(GPR_DEBUG, "bidirectional_stream_read(%p)", s->cbs); stream_state->state_op_done[OP_READ_REQ_MADE] = true; /* Indicates that at least one read request has been made */ - bidirectional_stream_read(s->cbs, stream_state->rs.read_buffer, - stream_state->rs.remaining_bytes); - stream_state->pending_read_from_cronet = true; + read_grpc_header(s); result = ACTION_TAKEN_NO_CALLBACK; } } else if (stream_state->rs.remaining_bytes == 0) { @@ -1316,15 +1318,8 @@ static enum e_op_result execute_stream_op(struct op_and_state* oas) { oas->state.state_op_done[OP_RECV_MESSAGE] = true; /* Do an extra read to trigger on_succeeded() callback in case connection is closed */ - stream_state->rs.read_buffer = stream_state->rs.grpc_header_bytes; - stream_state->rs.compressed = false; - stream_state->rs.received_bytes = 0; - stream_state->rs.remaining_bytes = GRPC_HEADER_SIZE_IN_BYTES; stream_state->rs.length_field_received = false; - CRONET_LOG(GPR_DEBUG, "bidirectional_stream_read(%p)", s->cbs); - bidirectional_stream_read(s->cbs, stream_state->rs.read_buffer, - stream_state->rs.remaining_bytes); - stream_state->pending_read_from_cronet = true; + read_grpc_header(s); result = ACTION_TAKEN_NO_CALLBACK; } } else if (stream_op->recv_trailing_metadata && From 43a574d1ae049ce92a6e619785d16bf7367d6562 Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Fri, 12 Jun 2020 12:59:26 -0700 Subject: [PATCH 44/80] Trust google-auth library will handle their dependency properly --- requirements.bazel.txt | 4 +--- requirements.txt | 2 -- src/python/grpcio_tests/setup.py | 2 +- tools/run_tests/helper_scripts/build_python.sh | 7 ++----- 4 files changed, 4 insertions(+), 11 deletions(-) diff --git a/requirements.bazel.txt b/requirements.bazel.txt index 0e2ecbe0907..10b47550f99 100644 --- a/requirements.bazel.txt +++ b/requirements.bazel.txt @@ -6,7 +6,7 @@ protobuf>=3.5.0.post1 six>=1.10 wheel>=0.29 futures>=2.2.0 -google-auth>=1.17.1 +google-auth>=1.17.2 oauth2client==4.1.0 requests>=2.14.2 urllib3>=1.23 @@ -14,5 +14,3 @@ chardet==3.0.4 certifi==2017.4.17 idna==2.7 googleapis-common-protos==1.5.5 -# rsa 4.3 is the last version support Python 2 -rsa==4.3 diff --git a/requirements.txt b/requirements.txt index 773e2c7c79b..27dd7d9f63b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,5 +5,3 @@ enum34>=1.0.4 protobuf>=3.5.0.post1 six>=1.10 wheel>=0.29 -# rsa 4.3 is the last version support Python 2 -rsa==4.3 diff --git a/src/python/grpcio_tests/setup.py b/src/python/grpcio_tests/setup.py index 67d22fd7148..6c1ed2f25b9 100644 --- a/src/python/grpcio_tests/setup.py +++ b/src/python/grpcio_tests/setup.py @@ -44,7 +44,7 @@ INSTALL_REQUIRES = ( 'grpcio-tools>={version}'.format(version=grpc_version.VERSION), 'grpcio-health-checking>={version}'.format(version=grpc_version.VERSION), 'oauth2client>=1.4.7', 'protobuf>=3.6.0', 'six>=1.10', - 'google-auth>=1.17.1', 'requests>=2.14.2') + 'google-auth>=1.17.2', 'requests>=2.14.2') if not PY3: INSTALL_REQUIRES += ('futures>=2.2.0',) diff --git a/tools/run_tests/helper_scripts/build_python.sh b/tools/run_tests/helper_scripts/build_python.sh index 699aa849972..25efd7170ee 100755 --- a/tools/run_tests/helper_scripts/build_python.sh +++ b/tools/run_tests/helper_scripts/build_python.sh @@ -222,13 +222,10 @@ pip_install_dir "$ROOT/src/python/grpcio_status" # Install testing pip_install_dir "$ROOT/src/python/grpcio_testing" -# rsa 4.3 is the latest version support Python 2 -pip_install rsa==4.3 - # Build/install tests pip_install coverage==4.4 oauth2client==4.1.0 \ - google-auth==1.17.1 requests==2.14.2 \ - googleapis-common-protos==1.5.5 + google-auth>=1.17.2 requests==2.14.2 \ + googleapis-common-protos>=1.5.5 $VENV_PYTHON "$ROOT/src/python/grpcio_tests/setup.py" preprocess $VENV_PYTHON "$ROOT/src/python/grpcio_tests/setup.py" build_package_protos pip_install_dir "$ROOT/src/python/grpcio_tests" From 815bd8b90f1b89702b65caf187a7f3919f29b75d Mon Sep 17 00:00:00 2001 From: srini100 Date: Mon, 15 Jun 2020 13:44:47 -0700 Subject: [PATCH 45/80] resolving comment --- doc/grpc_xds_features.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/grpc_xds_features.md b/doc/grpc_xds_features.md index 7e0921c88c4..3537db2b965 100644 --- a/doc/grpc_xds_features.md +++ b/doc/grpc_xds_features.md @@ -27,7 +27,7 @@ graceful failover themselves. The EDS policy will not support per-endpoint stats; it will report only per-locality stats. -An [`lb_endpoint`](https://github.com/envoyproxy/envoy/blob/master/api/envoy/api/v2/endpoint/endpoint_components.proto) +An [`lb_endpoint`](https://github.com/envoyproxy/envoy/blob/12a4bc430eaf440ceb0d11286cfbd4c16b79cdd1/api/envoy/api/v2/endpoint/endpoint_components.proto#L72) is ignored if the `health_status` is not HEALTHY or UNKNOWN. The optional `load_balancing_weight` is always ignored. @@ -36,5 +36,5 @@ to authenticate with the xDS server. Features | gRFCs | [C++, Python,
    Ruby, PHP, C#](https://github.com/grpc/grpc/releases) | [Java](https://github.com/grpc/grpc-java/releases) | [Go](https://github.com/grpc/grpc-go/releases) ---------|--------|--------------|------|------ -**xDS Infrastructure:**
    LDS->RDS->CDS->EDS flow,
    ADS stream,
    Client-side Load reporting via [LRS](https://github.com/envoyproxy/data-plane-api/blob/master/envoy/service/load_stats/v2/lrs.proto), | [A27](https://github.com/grpc/proposal/blob/master/A27-xds-global-load-balancing.md) | v1.30.0 | v1.30.0 | v1.30.0 | -**Load Balancing:**
    Virtual host matching,
    Only default path ("" or "/") matching,
    Priority-based weighted round-robin locality picking,
    Round-robin endpoint picking within locality,
    Cluster route action| [A27](https://github.com/grpc/proposal/blob/master/A27-xds-global-load-balancing.md) | v1.30.0 | v1.30.0 | v1.30.0 | +**xDS Infrastructure in gRPC client channel:**
    LDS->RDS->CDS->EDS flow,
    ADS stream, | [A27](https://github.com/grpc/proposal/blob/master/A27-xds-global-load-balancing.md) | v1.30.0 | v1.30.0 | v1.30.0 | +**Load Balancing:**
    Virtual host matching,
    Only default path ("" or "/") matching,
    Priority-based weighted round-robin locality picking,
    Round-robin endpoint picking within locality,
    Cluster route action,
    Client-side Load reporting via [LRS](https://github.com/envoyproxy/data-plane-api/blob/master/envoy/service/load_stats/v2/lrs.proto)| [A27](https://github.com/grpc/proposal/blob/master/A27-xds-global-load-balancing.md) | v1.30.0 | v1.30.0 | v1.30.0 | From 18054754c5d85213bba6f8c77bb3587bcc1b7dc4 Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Mon, 15 Jun 2020 14:19:54 -0700 Subject: [PATCH 46/80] Install rsa==4.0 to avoid the egg cache race issue --- tools/run_tests/helper_scripts/build_python.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/helper_scripts/build_python.sh b/tools/run_tests/helper_scripts/build_python.sh index 25efd7170ee..60031fa6e95 100755 --- a/tools/run_tests/helper_scripts/build_python.sh +++ b/tools/run_tests/helper_scripts/build_python.sh @@ -225,7 +225,7 @@ pip_install_dir "$ROOT/src/python/grpcio_testing" # Build/install tests pip_install coverage==4.4 oauth2client==4.1.0 \ google-auth>=1.17.2 requests==2.14.2 \ - googleapis-common-protos>=1.5.5 + googleapis-common-protos>=1.5.5 rsa==4.0 $VENV_PYTHON "$ROOT/src/python/grpcio_tests/setup.py" preprocess $VENV_PYTHON "$ROOT/src/python/grpcio_tests/setup.py" build_package_protos pip_install_dir "$ROOT/src/python/grpcio_tests" From c9acb661852ed423986fb3519e6b22fb18ac8a56 Mon Sep 17 00:00:00 2001 From: Srini Polavarapu Date: Mon, 15 Jun 2020 14:58:54 -0700 Subject: [PATCH 47/80] regenerate projects --- tools/doxygen/Doxyfile.c++ | 1 + tools/doxygen/Doxyfile.c++.internal | 1 + tools/doxygen/Doxyfile.core | 1 + tools/doxygen/Doxyfile.core.internal | 1 + tools/doxygen/Doxyfile.objc | 1 + tools/doxygen/Doxyfile.objc.internal | 1 + 6 files changed, 6 insertions(+) diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index f9fcb6142ee..bf68c587f56 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -778,6 +778,7 @@ doc/fail_fast.md \ doc/fork_support.md \ doc/g_stands_for.md \ doc/grpc_release_schedule.md \ +doc/grpc_xds_features.md \ doc/health-checking.md \ doc/http-grpc-status-mapping.md \ doc/http2-interop-test-descriptions.md \ diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 613e824f4d7..1867476c484 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -778,6 +778,7 @@ doc/fail_fast.md \ doc/fork_support.md \ doc/g_stands_for.md \ doc/grpc_release_schedule.md \ +doc/grpc_xds_features.md \ doc/health-checking.md \ doc/http-grpc-status-mapping.md \ doc/http2-interop-test-descriptions.md \ diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index 5242b54e13a..8da50b0c21b 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -785,6 +785,7 @@ doc/fail_fast.md \ doc/fork_support.md \ doc/g_stands_for.md \ doc/grpc_release_schedule.md \ +doc/grpc_xds_features.md \ doc/health-checking.md \ doc/http-grpc-status-mapping.md \ doc/http2-interop-test-descriptions.md \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index b0a051f81b8..bff4f23c4a4 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -785,6 +785,7 @@ doc/fail_fast.md \ doc/fork_support.md \ doc/g_stands_for.md \ doc/grpc_release_schedule.md \ +doc/grpc_xds_features.md \ doc/health-checking.md \ doc/http-grpc-status-mapping.md \ doc/http2-interop-test-descriptions.md \ diff --git a/tools/doxygen/Doxyfile.objc b/tools/doxygen/Doxyfile.objc index 2eace4b5928..0eea44a5c24 100644 --- a/tools/doxygen/Doxyfile.objc +++ b/tools/doxygen/Doxyfile.objc @@ -776,6 +776,7 @@ doc/fail_fast.md \ doc/fork_support.md \ doc/g_stands_for.md \ doc/grpc_release_schedule.md \ +doc/grpc_xds_features.md \ doc/health-checking.md \ doc/http-grpc-status-mapping.md \ doc/http2-interop-test-descriptions.md \ diff --git a/tools/doxygen/Doxyfile.objc.internal b/tools/doxygen/Doxyfile.objc.internal index 93a25087e4d..3c4593c6d1a 100644 --- a/tools/doxygen/Doxyfile.objc.internal +++ b/tools/doxygen/Doxyfile.objc.internal @@ -776,6 +776,7 @@ doc/fail_fast.md \ doc/fork_support.md \ doc/g_stands_for.md \ doc/grpc_release_schedule.md \ +doc/grpc_xds_features.md \ doc/health-checking.md \ doc/http-grpc-status-mapping.md \ doc/http2-interop-test-descriptions.md \ From 52e1059a6d086f7a71ccca870d439319db8adfa0 Mon Sep 17 00:00:00 2001 From: Donna Dionne Date: Mon, 15 Jun 2020 18:12:09 -0700 Subject: [PATCH 48/80] Passing repo to markdroth --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- .github/ISSUE_TEMPLATE/cleanup_request.md | 2 +- .github/ISSUE_TEMPLATE/feature_request.md | 2 +- .github/ISSUE_TEMPLATE/question.md | 2 +- .github/pull_request_template.md | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index dbec90aefc9..663fcbde3fb 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -2,7 +2,7 @@ name: Report a bug about: Create a report to help us improve labels: kind/bug, priority/P2 -assignees: veblush +assignees: markdroth --- diff --git a/.github/ISSUE_TEMPLATE/cleanup_request.md b/.github/ISSUE_TEMPLATE/cleanup_request.md index ef88ef49ac5..cd547e286c1 100644 --- a/.github/ISSUE_TEMPLATE/cleanup_request.md +++ b/.github/ISSUE_TEMPLATE/cleanup_request.md @@ -2,7 +2,7 @@ name: Request a cleanup about: Suggest a cleanup in our repository labels: kind/internal cleanup, priority/P2 -assignees: donnadionne +assignees: markdroth --- diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index f230fe25a3a..717752f9207 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -2,7 +2,7 @@ name: Request a feature about: Suggest an idea for this project labels: kind/enhancement, priority/P2 -assignees: donnadionne +assignees: markdroth --- diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md index 77ecf5ea984..48f55c83d73 100644 --- a/.github/ISSUE_TEMPLATE/question.md +++ b/.github/ISSUE_TEMPLATE/question.md @@ -2,7 +2,7 @@ name: Ask a question about: Ask a question labels: kind/question, priority/P3 -assignees: donnadionne +assignees: markdroth --- diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index eddba6676da..a85cfad9c7a 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -8,4 +8,4 @@ If you know who should review your pull request, please remove the mentioning be --> -@donnadionne +@markdroth From 23e1d30327923ac93beaeb6cdcfb0eef9be759c4 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Thu, 30 Jan 2020 16:27:27 -0800 Subject: [PATCH 49/80] Removes manylinux1-specific instructions --- include/grpc/impl/codegen/port_platform.h | 25 ------------------- src/core/lib/iomgr/port.h | 13 ---------- tools/run_tests/artifacts/artifact_targets.py | 10 ++------ 3 files changed, 2 insertions(+), 46 deletions(-) diff --git a/include/grpc/impl/codegen/port_platform.h b/include/grpc/impl/codegen/port_platform.h index 4f213ff5979..7436e644504 100644 --- a/include/grpc/impl/codegen/port_platform.h +++ b/include/grpc/impl/codegen/port_platform.h @@ -112,31 +112,6 @@ #define GPR_WINDOWS_ATOMIC 1 #define GPR_MSVC_TLS 1 #endif -#elif defined(GPR_MANYLINUX1) -// TODO(atash): manylinux1 is just another __linux__ but with ancient -// libraries; it should be integrated with the `__linux__` definitions below. -#define GPR_PLATFORM_STRING "manylinux" -#define GPR_POSIX_CRASH_HANDLER 1 -#define GPR_CPU_POSIX 1 -#define GPR_GCC_ATOMIC 1 -#define GPR_GCC_TLS 1 -#define GPR_LINUX 1 -#define GPR_LINUX_LOG 1 -#define GPR_SUPPORT_CHANNELS_FROM_FD 1 -#define GPR_LINUX_ENV 1 -#define GPR_POSIX_TMPFILE 1 -#define GPR_POSIX_STRING 1 -#define GPR_POSIX_SUBPROCESS 1 -#define GPR_POSIX_SYNC 1 -#define GPR_POSIX_TIME 1 -#define GPR_HAS_PTHREAD_H 1 -#define GPR_GETPID_IN_UNISTD_H 1 -#ifdef _LP64 -#define GPR_ARCH_64 1 -#else /* _LP64 */ -#define GPR_ARCH_32 1 -#endif /* _LP64 */ -#include #elif defined(ANDROID) || defined(__ANDROID__) #define GPR_PLATFORM_STRING "android" #define GPR_ANDROID 1 diff --git a/src/core/lib/iomgr/port.h b/src/core/lib/iomgr/port.h index 211423e643d..02f64f318a9 100644 --- a/src/core/lib/iomgr/port.h +++ b/src/core/lib/iomgr/port.h @@ -33,19 +33,6 @@ #endif #if defined(GRPC_CUSTOM_SOCKET) // Do Nothing -#elif defined(GPR_MANYLINUX1) -#define GRPC_HAVE_ARPA_NAMESER 1 -#define GRPC_HAVE_IFADDRS 1 -#define GRPC_HAVE_IPV6_RECVPKTINFO 1 -#define GRPC_HAVE_IP_PKTINFO 1 -#define GRPC_HAVE_MSG_NOSIGNAL 1 -#define GRPC_HAVE_UNIX_SOCKET 1 -#define GRPC_POSIX_FORK 1 -#define GRPC_POSIX_NO_SPECIAL_WAKEUP_FD 1 -#define GRPC_POSIX_SOCKET 1 -#define GRPC_POSIX_SOCKETUTILS 1 -#define GRPC_POSIX_WAKEUP_FD 1 -#define GRPC_LINUX_EPOLL 1 #elif defined(GPR_WINDOWS) #define GRPC_WINSOCK_SOCKET 1 #define GRPC_WINDOWS_SOCKETUTILS 1 diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index f748a9a909a..bb21eafca3e 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -144,14 +144,8 @@ class PythonArtifact: environ['PYTHON'] = '/opt/python/{}/bin/python'.format( self.py_version) environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.py_version) - # Platform autodetection for the manylinux1 image breaks so we set the - # defines ourselves. - # TODO(atash) get better platform-detection support in core so we don't - # need to do this manually... - environ['CFLAGS'] = '-DGPR_MANYLINUX1=1' environ['GRPC_BUILD_GRPCIO_TOOLS_DEPENDENTS'] = 'TRUE' environ['GRPC_BUILD_MANYLINUX_WHEEL'] = 'TRUE' - return create_docker_jobspec( self.name, # NOTE(rbellevi): Do *not* update this without also ensuring the @@ -161,8 +155,8 @@ class PythonArtifact: 'tools/run_tests/artifacts/build_artifact_python.sh', environ=environ, timeout_seconds=60 * 60, - docker_base_image='quay.io/pypa/manylinux1_i686' - if self.arch == 'x86' else 'quay.io/pypa/manylinux1_x86_64') + docker_base_image='quay.io/pypa/{}_{}'.format( + self.platform, 'x86' if self.arch == 'x86' else 'x64')) elif self.platform == 'windows': if 'Python27' in self.py_version: environ['EXT_COMPILER'] = 'mingw32' From be1bc912fd18a59bb61d6690b213ec27e72f1b21 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Thu, 21 May 2020 09:21:23 -0700 Subject: [PATCH 50/80] Remove docker base image for manylinux --- tools/run_tests/artifacts/artifact_targets.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index bb21eafca3e..dbc37e6f57c 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -154,9 +154,7 @@ class PythonArtifact: (self.platform, self.arch), 'tools/run_tests/artifacts/build_artifact_python.sh', environ=environ, - timeout_seconds=60 * 60, - docker_base_image='quay.io/pypa/{}_{}'.format( - self.platform, 'x86' if self.arch == 'x86' else 'x64')) + timeout_seconds=60 * 60) elif self.platform == 'windows': if 'Python27' in self.py_version: environ['EXT_COMPILER'] = 'mingw32' From 005dbafe6652a387e15e882d4ea757a61642d246 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 16 Jun 2020 09:58:56 +0200 Subject: [PATCH 51/80] fix mac tests on kokoro workers --- src/csharp/Grpc.Tools.Tests/MsBuildAssemblyHelper.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/csharp/Grpc.Tools.Tests/MsBuildAssemblyHelper.cs b/src/csharp/Grpc.Tools.Tests/MsBuildAssemblyHelper.cs index c24cddb2348..8ca941b7e12 100644 --- a/src/csharp/Grpc.Tools.Tests/MsBuildAssemblyHelper.cs +++ b/src/csharp/Grpc.Tools.Tests/MsBuildAssemblyHelper.cs @@ -44,6 +44,14 @@ namespace Grpc.Tools.Tests // Construct the location of MsBuild assemblies from the location of mscorlib assembly. var msbuildToolPath = Path.Combine(mscorlibDir, "..", "msbuild", "Current", "bin"); + if (!Directory.Exists(msbuildToolPath)) + { + // with older versions of mono for Mac (e.g. mono 5.16.0 which is currently + // installed on the kokoro mac workers) the "Current" symlink doesn't exist + // so also try specifying the msbuild version explicitly + msbuildToolPath = Path.Combine(mscorlibDir, "..", "msbuild", "15.0", "bin"); + } + // To make sure we've constructed the right path, make sure the assemblies we're interested // in are there. foreach(var assemblyName in new [] {"Microsoft.Build.Framework.dll", "Microsoft.Build.Utilities.v4.0.dll", "Microsoft.Build.Utilities.Core.dll"}) From 4c4933e056cb2c1889c51242fa8317ea268083df Mon Sep 17 00:00:00 2001 From: Patrice Chalin Date: Tue, 16 Jun 2020 08:40:59 -0400 Subject: [PATCH 52/80] Docs: fix links to quick start pages --- doc/python/sphinx/grpc.rst | 2 +- examples/cpp/README.md | 2 +- examples/cpp/helloworld/README.md | 2 +- examples/python/README.md | 2 +- examples/python/helloworld/README.md | 2 +- src/php/README.md | 32 ++++++++++++++-------------- src/ruby/README.md | 2 +- 7 files changed, 22 insertions(+), 22 deletions(-) diff --git a/doc/python/sphinx/grpc.rst b/doc/python/sphinx/grpc.rst index 67867a683e9..665392f28d0 100644 --- a/doc/python/sphinx/grpc.rst +++ b/doc/python/sphinx/grpc.rst @@ -6,7 +6,7 @@ gRPC Tutorial -------- -If you want to see gRPC in action first, visit the `Python Quickstart `_. +If you want to see gRPC in action first, visit the `Python Quickstart `_. Or, if you would like dive in with more extensive usage of gRPC Python, check `gRPC Basics - Python `_ out. diff --git a/examples/cpp/README.md b/examples/cpp/README.md index a0ff629b1a0..5254b30c832 100644 --- a/examples/cpp/README.md +++ b/examples/cpp/README.md @@ -9,5 +9,5 @@ README files. [gRPC Basics]: https://grpc.io/docs/tutorials/basic/cpp [Hello World]: helloworld -[Quick Start]: https://grpc.io/docs/quickstart/cpp +[Quick Start]: https://grpc.io/docs/languages/cpp/quickstart [Route Guide]: route_guide diff --git a/examples/cpp/helloworld/README.md b/examples/cpp/helloworld/README.md index e13c83281a7..a06cc8e3cb2 100644 --- a/examples/cpp/helloworld/README.md +++ b/examples/cpp/helloworld/README.md @@ -3,4 +3,4 @@ You can find a complete set of instructions for building gRPC and running the Hello World app in the [C++ Quick Start][]. -[C++ Quick Start]: https://grpc.io/docs/quickstart/cpp +[C++ Quick Start]: https://grpc.io/docs/languages/cpp/quickstart diff --git a/examples/python/README.md b/examples/python/README.md index 63d61e439b6..be57d89785d 100644 --- a/examples/python/README.md +++ b/examples/python/README.md @@ -1 +1 @@ -[This code's documentation lives on the grpc.io site.](https://grpc.io/docs/quickstart/python.html) +[This code's documentation lives on the grpc.io site.](https://grpc.io/docs/languages/python/quickstart) diff --git a/examples/python/helloworld/README.md b/examples/python/helloworld/README.md index 63d61e439b6..be57d89785d 100644 --- a/examples/python/helloworld/README.md +++ b/examples/python/helloworld/README.md @@ -1 +1 @@ -[This code's documentation lives on the grpc.io site.](https://grpc.io/docs/quickstart/python.html) +[This code's documentation lives on the grpc.io site.](https://grpc.io/docs/languages/python/quickstart) diff --git a/src/php/README.md b/src/php/README.md index 9aa66896684..e3723cd2faf 100644 --- a/src/php/README.md +++ b/src/php/README.md @@ -3,7 +3,7 @@ This directory contains source code for PHP implementation of gRPC layered on shared C library. The same installation guides with more examples and -tutorials can be seen at [grpc.io](https://grpc.io/docs/quickstart/php.html). +tutorials can be seen at [grpc.io](https://grpc.io/docs/languages/php/quickstart). gRPC PHP installation instructions for Google Cloud Platform is in [cloud.google.com](https://cloud.google.com/php/grpc). @@ -36,7 +36,7 @@ or specific version $ [sudo] pecl install grpc-1.25.0 ``` -Note: for users on CentOS/RHEL 6, unfortunately this step won’t work. +Note: for users on CentOS/RHEL 6, unfortunately this step won’t work. Please follow the instructions below to compile the PECL extension from source. @@ -74,8 +74,8 @@ $ make $ [sudo] make install ``` -This will compile and install the `grpc` extension into the -standard PHP extension directory. You should be able to run +This will compile and install the `grpc` extension into the +standard PHP extension directory. You should be able to run the [unit tests](#unit-tests), with the `grpc` extension installed. @@ -100,7 +100,7 @@ composer package as well. Add this to your project's `composer.json` file. } ``` -To run tests with generated stub code from `.proto` files, you will also +To run tests with generated stub code from `.proto` files, you will also need the `composer` and `protoc` binaries. You can find out how to get these below. @@ -120,7 +120,7 @@ classes. If you don't have it already, you need to install the protobuf compiler `protoc`, version 3.5.0+ (the newer the better) for the current gRPC version. -If you installed already, make the protobuf version is compatible to the +If you installed already, make the protobuf version is compatible to the grpc version you installed. If you build grpc.so from the souce, you can check the version of grpc inside package.xml file. @@ -128,15 +128,15 @@ The compatibility between the grpc and protobuf version is listed as table below: grpc | protobuf ---- | --- +--- | --- v1.0.0 | 3.0.0(GA) v1.0.1 | 3.0.2 -v1.1.0 | 3.1.0 -v1.2.0 | 3.2.0 -v1.2.0 | 3.2.0 -v1.3.4 | 3.3.0 +v1.1.0 | 3.1.0 +v1.2.0 | 3.2.0 +v1.2.0 | 3.2.0 +v1.3.4 | 3.3.0 v1.3.5 | 3.2.0 -v1.4.0 | 3.3.0 +v1.4.0 | 3.3.0 v1.6.0 | 3.4.0 v1.8.0 | 3.5.0 v1.12.0 | 3.5.2 @@ -197,7 +197,7 @@ version you build this plugin. ### `protobuf` runtime library There are two `protobuf` runtime libraries to choose from. They are identical -in terms of APIs offered. The C implementation provides better performance, +in terms of APIs offered. The C implementation provides better performance, while the native implementation is easier to install. #### C implementation (for better performance) @@ -365,11 +365,11 @@ $client = new Helloworld\GreeterClient('localhost:50051', [ ]); ``` -### Compression +### Compression You can customize the compression behavior on the client side, by specifying the following options when constructing your PHP client. -``` +``` Possible values for grpc.default_compression_algorithm: 0 - No compression 1 - Compress with DEFLATE algorithm @@ -387,7 +387,7 @@ Here's an example on how you can put them all together: ``` $client = new Helloworld\GreeterClient('localhost:50051', [ 'credentials' => Grpc\ChannelCredentials::createInsecure(), - 'grpc.default_compression_algorithm' => 2, + 'grpc.default_compression_algorithm' => 2, 'grpc.default_compression_level' => 2, ]); diff --git a/src/ruby/README.md b/src/ruby/README.md index f6fce3ed22e..8187d7f3b5c 100644 --- a/src/ruby/README.md +++ b/src/ruby/README.md @@ -74,5 +74,5 @@ Directory structure is the layout for [ruby extensions][] [ruby extensions]:http://guides.rubygems.org/gems-with-extensions/ [rubydoc]: http://www.rubydoc.info/gems/grpc -[grpc.io]: https://grpc.io/docs/quickstart/ruby.html +[grpc.io]: https://grpc.io/docs/languages/ruby/quickstart [Debian jessie-backports]:http://backports.debian.org/Instructions/ From 5ac3aa789f28c50419cdbf06981cb429998be9c9 Mon Sep 17 00:00:00 2001 From: Patrice Chalin Date: Tue, 16 Jun 2020 08:47:24 -0400 Subject: [PATCH 53/80] Docs: fix links to grpc.io tutorial pages --- doc/python/sphinx/grpc.rst | 2 +- examples/cpp/README.md | 2 +- examples/cpp/route_guide/README.md | 2 +- examples/csharp/Helloworld/README.md | 2 +- examples/csharp/HelloworldLegacyCsproj/README.md | 4 ++-- examples/csharp/RouteGuide/README.md | 2 +- examples/node/README.md | 2 +- examples/node/dynamic_codegen/route_guide/README.md | 2 +- examples/node/static_codegen/route_guide/README.md | 2 +- examples/objective-c/auth_sample/README.md | 2 +- examples/objective-c/helloworld/README.md | 2 +- examples/objective-c/route_guide/README.md | 2 +- examples/php/README.md | 2 +- examples/php/route_guide/README.md | 2 +- examples/python/metadata/README.md | 2 +- examples/python/multiplex/README.md | 2 +- examples/python/route_guide/README.md | 2 +- examples/ruby/README.md | 2 +- examples/ruby/route_guide/README.md | 2 +- src/cpp/README.md | 4 ++-- src/csharp/README.md | 12 ++++++------ 21 files changed, 28 insertions(+), 28 deletions(-) diff --git a/doc/python/sphinx/grpc.rst b/doc/python/sphinx/grpc.rst index 665392f28d0..4717eb12332 100644 --- a/doc/python/sphinx/grpc.rst +++ b/doc/python/sphinx/grpc.rst @@ -7,7 +7,7 @@ Tutorial -------- If you want to see gRPC in action first, visit the `Python Quickstart `_. -Or, if you would like dive in with more extensive usage of gRPC Python, check `gRPC Basics - Python `_ out. +Or, if you would like dive in with more extensive usage of gRPC Python, check `gRPC Basics - Python `_ out. Example diff --git a/examples/cpp/README.md b/examples/cpp/README.md index 5254b30c832..7c281de30c0 100644 --- a/examples/cpp/README.md +++ b/examples/cpp/README.md @@ -7,7 +7,7 @@ For information about the other examples in this directory, see their respective README files. -[gRPC Basics]: https://grpc.io/docs/tutorials/basic/cpp +[gRPC Basics]: https://grpc.io/docs/languages/cpp/basics [Hello World]: helloworld [Quick Start]: https://grpc.io/docs/languages/cpp/quickstart [Route Guide]: route_guide diff --git a/examples/cpp/route_guide/README.md b/examples/cpp/route_guide/README.md index 1d1956d275d..2b2376ca180 100644 --- a/examples/cpp/route_guide/README.md +++ b/examples/cpp/route_guide/README.md @@ -3,4 +3,4 @@ The files in this folder are the samples used in [gRPC Basics: C++][], a detailed tutorial for using gRPC in C++. -[gRPC Basics: C++]:https://grpc.io/docs/tutorials/basic/cpp +[gRPC Basics: C++]:https://grpc.io/docs/languages/cpp/basics diff --git a/examples/csharp/Helloworld/README.md b/examples/csharp/Helloworld/README.md index e4771ee91a1..8897d4fc00c 100644 --- a/examples/csharp/Helloworld/README.md +++ b/examples/csharp/Helloworld/README.md @@ -36,4 +36,4 @@ Tutorial You can find a more detailed tutorial about Grpc in [gRPC Basics: C#][] [helloworld.proto]:../../protos/helloworld.proto -[gRPC Basics: C#]:https://grpc.io/docs/tutorials/basic/csharp.html +[gRPC Basics: C#]:https://grpc.io/docs/languages/csharp/basics diff --git a/examples/csharp/HelloworldLegacyCsproj/README.md b/examples/csharp/HelloworldLegacyCsproj/README.md index 4435faeb086..3f3889fe216 100644 --- a/examples/csharp/HelloworldLegacyCsproj/README.md +++ b/examples/csharp/HelloworldLegacyCsproj/README.md @@ -40,7 +40,7 @@ download all of the NuGet dependencies of gRPC. Using these IDEs, a workaround is as follows: * Obtain a nuget executable for your platform and update it with - `nuget update -self`. + `nuget update -self`. * Navigate to this directory and run `nuget restore`. * Now that packages have been restored into their proper package folder, build the solution from your IDE. @@ -71,4 +71,4 @@ Tutorial You can find a more detailed tutorial in [gRPC Basics: C#][] [helloworld.proto]:../../protos/helloworld.proto -[gRPC Basics: C#]:https://grpc.io/docs/tutorials/basic/csharp.html +[gRPC Basics: C#]:https://grpc.io/docs/languages/csharp/basics diff --git a/examples/csharp/RouteGuide/README.md b/examples/csharp/RouteGuide/README.md index 3cfb14ae9a9..bfdb9e14f06 100644 --- a/examples/csharp/RouteGuide/README.md +++ b/examples/csharp/RouteGuide/README.md @@ -3,4 +3,4 @@ The files in this folder are the samples used in [gRPC Basics: C#][], a detailed tutorial for using gRPC in C#. -[gRPC Basics: C#]:https://grpc.io/docs/tutorials/basic/csharp.html +[gRPC Basics: C#]:https://grpc.io/docs/languages/csharp/basics diff --git a/examples/node/README.md b/examples/node/README.md index 0264e9d66b7..b45488b8b56 100644 --- a/examples/node/README.md +++ b/examples/node/README.md @@ -47,4 +47,4 @@ TUTORIAL You can find a more detailed tutorial in [gRPC Basics: Node.js][] [Install gRPC Node]:../../src/node -[gRPC Basics: Node.js]:https://grpc.io/docs/tutorials/basic/node.html +[gRPC Basics: Node.js]:https://grpc.io/docs/languages/node/basics diff --git a/examples/node/dynamic_codegen/route_guide/README.md b/examples/node/dynamic_codegen/route_guide/README.md index 7d6b0b7debe..fcd147054a6 100644 --- a/examples/node/dynamic_codegen/route_guide/README.md +++ b/examples/node/dynamic_codegen/route_guide/README.md @@ -2,4 +2,4 @@ The files in this folder are the samples used in [gRPC Basics: Node.js][], a detailed tutorial for using gRPC in Node.js. -[gRPC Basics: Node.js]:https://grpc.io/docs/tutorials/basic/node.html +[gRPC Basics: Node.js]:https://grpc.io/docs/languages/node/basics diff --git a/examples/node/static_codegen/route_guide/README.md b/examples/node/static_codegen/route_guide/README.md index 7d6b0b7debe..fcd147054a6 100644 --- a/examples/node/static_codegen/route_guide/README.md +++ b/examples/node/static_codegen/route_guide/README.md @@ -2,4 +2,4 @@ The files in this folder are the samples used in [gRPC Basics: Node.js][], a detailed tutorial for using gRPC in Node.js. -[gRPC Basics: Node.js]:https://grpc.io/docs/tutorials/basic/node.html +[gRPC Basics: Node.js]:https://grpc.io/docs/languages/node/basics diff --git a/examples/objective-c/auth_sample/README.md b/examples/objective-c/auth_sample/README.md index a0063de3f2f..6e90be39b16 100644 --- a/examples/objective-c/auth_sample/README.md +++ b/examples/objective-c/auth_sample/README.md @@ -1,3 +1,3 @@ # OAuth2 on gRPC: Objective-C -This is the supporting code for the tutorial "[OAuth2 on gRPC: Objective-C](https://grpc.io/docs/tutorials/auth/oauth2-objective-c.html)." +This is the supporting code for the tutorial "[OAuth2 on gRPC: Objective-C](https://grpc.io/docs/languages/objective-c/oauth2)." diff --git a/examples/objective-c/helloworld/README.md b/examples/objective-c/helloworld/README.md index c57e07ca931..2bb6d6c7b6d 100644 --- a/examples/objective-c/helloworld/README.md +++ b/examples/objective-c/helloworld/README.md @@ -104,4 +104,4 @@ $ bazel run :HelloWorld --ios_simulator_version='' --ios_sumlator_devic ## Tutorial -You can find a more detailed tutorial in [gRPC Basics: Objective-C](https://grpc.io/docs/tutorials/basic/objective-c.html). +You can find a more detailed tutorial in [gRPC Basics: Objective-C](https://grpc.io/docs/languages/objective/basics-c.html). diff --git a/examples/objective-c/route_guide/README.md b/examples/objective-c/route_guide/README.md index 60e5304651a..d32f1699cb9 100644 --- a/examples/objective-c/route_guide/README.md +++ b/examples/objective-c/route_guide/README.md @@ -1,4 +1,4 @@ # gRPC Basics: Objective-C -This is the supporting code for the tutorial "[gRPC Basics: Objective-C](https://grpc.io/docs/tutorials/basic/objective-c.html)." +This is the supporting code for the tutorial "[gRPC Basics: Objective-C](https://grpc.io/docs/languages/objective/basics-c.html)." diff --git a/examples/php/README.md b/examples/php/README.md index 75b9e6fb829..2182e39628a 100644 --- a/examples/php/README.md +++ b/examples/php/README.md @@ -53,4 +53,4 @@ This requires `php` >= 5.5, `pecl`, `composer` You can find a more detailed tutorial in [gRPC Basics: PHP][] [Node]:https://github.com/grpc/grpc/tree/master/examples/node -[gRPC Basics: PHP]:https://grpc.io/docs/tutorials/basic/php.html +[gRPC Basics: PHP]:https://grpc.io/docs/languages/php/basics diff --git a/examples/php/route_guide/README.md b/examples/php/route_guide/README.md index 6bea70dea03..e5abfbba053 100644 --- a/examples/php/route_guide/README.md +++ b/examples/php/route_guide/README.md @@ -3,4 +3,4 @@ The files in this folder are the samples used in [gRPC Basics: PHP][], a detailed tutorial for using gRPC in PHP. -[gRPC Basics: PHP]:https://grpc.io/docs/tutorials/basic/php.html +[gRPC Basics: PHP]:https://grpc.io/docs/languages/php/basics diff --git a/examples/python/metadata/README.md b/examples/python/metadata/README.md index 5aa75d504a8..26ffa44f6fa 100644 --- a/examples/python/metadata/README.md +++ b/examples/python/metadata/README.md @@ -2,5 +2,5 @@ An example showing how to add custom HTTP2 headers (or [metadata](https://grpc.i HTTP2 supports initial headers and trailing headers, which gRPC utilizes both of them ([learn more](https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md)). -More complete documentation lives at [grpc.io](https://grpc.io/docs/tutorials/basic/python.html). +More complete documentation lives at [grpc.io](https://grpc.io/docs/languages/python/basics). For API reference please see [API](https://grpc.io/grpc/python/grpc.html). diff --git a/examples/python/multiplex/README.md b/examples/python/multiplex/README.md index 2316cd39e88..8afdb4dffeb 100644 --- a/examples/python/multiplex/README.md +++ b/examples/python/multiplex/README.md @@ -1,3 +1,3 @@ An example showing two stubs sharing a channel and two servicers sharing a server. -More complete documentation lives at [grpc.io](https://grpc.io/docs/tutorials/basic/python.html). +More complete documentation lives at [grpc.io](https://grpc.io/docs/languages/python/basics). diff --git a/examples/python/route_guide/README.md b/examples/python/route_guide/README.md index 890e66ebad0..6f7a2f0fdf0 100644 --- a/examples/python/route_guide/README.md +++ b/examples/python/route_guide/README.md @@ -1 +1 @@ -[This code's documentation lives on the grpc.io site.](https://grpc.io/docs/tutorials/basic/python.html) +[This code's documentation lives on the grpc.io site.](https://grpc.io/docs/languages/python/basics) diff --git a/examples/ruby/README.md b/examples/ruby/README.md index c6af1a5a5e8..2a3dec338fe 100644 --- a/examples/ruby/README.md +++ b/examples/ruby/README.md @@ -60,4 +60,4 @@ You can find a more detailed tutorial in [gRPC Basics: Ruby][] [helloworld.proto]:../protos/helloworld.proto [RVM]:https://www.rvm.io/ [Install gRPC ruby]:../../src/ruby#installation -[gRPC Basics: Ruby]:https://grpc.io/docs/tutorials/basic/ruby.html +[gRPC Basics: Ruby]:https://grpc.io/docs/languages/ruby/basics diff --git a/examples/ruby/route_guide/README.md b/examples/ruby/route_guide/README.md index 12537c859e1..83501d4babf 100644 --- a/examples/ruby/route_guide/README.md +++ b/examples/ruby/route_guide/README.md @@ -3,4 +3,4 @@ The files in this folder are the samples used in [gRPC Basics: Ruby][], a detailed tutorial for using gRPC in Ruby. -[gRPC Basics: Ruby]:https://grpc.io/docs/tutorials/basic/ruby.html +[gRPC Basics: Ruby]:https://grpc.io/docs/languages/ruby/basics diff --git a/src/cpp/README.md b/src/cpp/README.md index 4f2dbc6014b..967a0a43b7f 100755 --- a/src/cpp/README.md +++ b/src/cpp/README.md @@ -167,10 +167,10 @@ documentation site at [grpc.io](https://grpc.io), specifically: * [Overview](https://grpc.io/docs): An introduction to gRPC with a simple Hello World example in all our supported languages, including C++. -* [gRPC Basics - C++](https://grpc.io/docs/tutorials/basic/cpp): +* [gRPC Basics - C++](https://grpc.io/docs/languages/cpp/basics): A tutorial that steps you through creating a simple gRPC C++ example application. -* [Asynchronous Basics - C++](https://grpc.io/docs/tutorials/async/helloasync-cpp): +* [Asynchronous Basics - C++](https://grpc.io/docs/languages/cpp/async): A tutorial that shows you how to use gRPC C++'s asynchronous/non-blocking APIs. diff --git a/src/csharp/README.md b/src/csharp/README.md index 089588254fd..f45d6e33706 100644 --- a/src/csharp/README.md +++ b/src/csharp/README.md @@ -18,7 +18,7 @@ The following documentation is for the original gRPC C# implementation only (the SUPPORTED PLATFORMS ------------------ -- [.NET Core](https://dotnet.github.io/) on Linux, Windows and Mac OS X +- [.NET Core](https://dotnet.github.io/) on Linux, Windows and Mac OS X - .NET Framework 4.5+ (Windows) - Mono 4+ on Linux, Windows and Mac OS X @@ -39,7 +39,7 @@ HOW TO USE - Open Visual Studio and start a new project/solution (alternatively, you can create a new project from command line with `dotnet` SDK) -- Add the [Grpc](https://www.nuget.org/packages/Grpc/) NuGet package as a dependency (Project options -> Manage NuGet Packages). +- Add the [Grpc](https://www.nuget.org/packages/Grpc/) NuGet package as a dependency (Project options -> Manage NuGet Packages). - To be able to generate code from Protocol Buffer (`.proto`) file definitions, add the [Grpc.Tools](https://www.nuget.org/packages/Grpc.Tools/) NuGet package which provides [code generation integrated into your build](BUILD-INTEGRATION.md). @@ -78,11 +78,11 @@ If you are a user of gRPC C#, go to Usage section above. **Windows, Linux or Mac OS X** - The easiest way to build is using the `run_tests.py` script that will take care of building the `grpc_csharp_ext` native library. - + ``` - # NOTE: make sure all necessary git submodules with dependencies + # NOTE: make sure all necessary git submodules with dependencies # are available by running "git submodule update --init" - + # from the gRPC repository root $ python tools/run_tests/run_tests.py -l csharp -c dbg --build_only ``` @@ -126,4 +126,4 @@ Internally, gRPC C# uses a native library written in C (gRPC C core) and invokes [API Reference]: https://grpc.io/grpc/csharp/api/Grpc.Core.html [Helloworld Example]: ../../examples/csharp/Helloworld -[RouteGuide Tutorial]: https://grpc.io/docs/tutorials/basic/csharp.html +[RouteGuide Tutorial]: https://grpc.io/docs/languages/csharp/basics From ff391fef8b6a9e1f855f035520819a02ddb5b24b Mon Sep 17 00:00:00 2001 From: Patrice Chalin Date: Tue, 16 Jun 2020 08:53:21 -0400 Subject: [PATCH 54/80] Fix broken link to Objective-C tutorial --- examples/objective-c/helloworld/README.md | 2 +- examples/objective-c/route_guide/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/objective-c/helloworld/README.md b/examples/objective-c/helloworld/README.md index 2bb6d6c7b6d..2fa390dc917 100644 --- a/examples/objective-c/helloworld/README.md +++ b/examples/objective-c/helloworld/README.md @@ -104,4 +104,4 @@ $ bazel run :HelloWorld --ios_simulator_version='' --ios_sumlator_devic ## Tutorial -You can find a more detailed tutorial in [gRPC Basics: Objective-C](https://grpc.io/docs/languages/objective/basics-c.html). +You can find a more detailed tutorial in [gRPC Basics: Objective-C](https://grpc.io/docs/languages/objective-c/basics). diff --git a/examples/objective-c/route_guide/README.md b/examples/objective-c/route_guide/README.md index d32f1699cb9..4cc57ceb58e 100644 --- a/examples/objective-c/route_guide/README.md +++ b/examples/objective-c/route_guide/README.md @@ -1,4 +1,4 @@ # gRPC Basics: Objective-C -This is the supporting code for the tutorial "[gRPC Basics: Objective-C](https://grpc.io/docs/languages/objective/basics-c.html)." +This is the supporting code for the tutorial "[gRPC Basics: Objective-C](https://grpc.io/docs/languages/objective-c/basics)." From 891ca8a4435a2100438b4099a2d7ee176bbfb05a Mon Sep 17 00:00:00 2001 From: Patrice Chalin Date: Tue, 16 Jun 2020 08:57:05 -0400 Subject: [PATCH 55/80] Guides links updates --- tools/run_tests/performance/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/performance/README.md b/tools/run_tests/performance/README.md index 6b41e97809c..1d0949882a9 100644 --- a/tools/run_tests/performance/README.md +++ b/tools/run_tests/performance/README.md @@ -1,7 +1,7 @@ # Overview of performance test suite, with steps for manual runs: For design of the tests, see -https://grpc.io/docs/guides/benchmarking.html. +https://grpc.io/docs/guides/benchmarking. ## Pre-reqs for running these manually: In general the benchmark workers and driver build scripts expect From 1dd7a01bc71c2279b0291017db886d14b6b5ed1d Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Tue, 16 Jun 2020 08:25:12 -0700 Subject: [PATCH 56/80] Avoid a use of ReleasableMutexLock --- src/cpp/server/server_context.cc | 55 ++++++++++++++++---------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/src/cpp/server/server_context.cc b/src/cpp/server/server_context.cc index 1c8e1e76829..21a13048bdd 100644 --- a/src/cpp/server/server_context.cc +++ b/src/cpp/server/server_context.cc @@ -174,31 +174,32 @@ void ServerContextBase::CompletionOp::FillOps(::grpc::internal::Call* call) { } bool ServerContextBase::CompletionOp::FinalizeResult(void** tag, bool* status) { - bool ret = false; - grpc_core::ReleasableMutexLock lock(&mu_); - if (done_intercepting_) { - /* We are done intercepting. */ - if (has_tag_) { - *tag = tag_; - ret = true; - } - Unref(); - return ret; - } - finalized_ = true; + // Decide whether to call the cancel callback within the lock + bool call_cancel; - // If for some reason the incoming status is false, mark that as a - // cancellation. - // TODO(vjpai): does this ever happen? - if (!*status) { - cancelled_ = 1; - } + { + grpc_core::MutexLock lock(&mu_); + if (done_intercepting_) { + // We are done intercepting. + bool has_tag = has_tag_; + if (has_tag) { + *tag = tag_; + } + Unref(); + return has_tag; + } + finalized_ = true; - // Decide whether to call the cancel callback before releasing the lock - bool call_cancel = (cancelled_ != 0); + // If for some reason the incoming status is false, mark that as a + // cancellation. + // TODO(vjpai): does this ever happen? + if (!*status) { + cancelled_ = 1; + } - // Release the lock since we may call a callback and interceptors now. - lock.Unlock(); + call_cancel = (cancelled_ != 0); + // Release the lock since we may call a callback and interceptors. + } if (call_cancel && callback_controller_ != nullptr) { callback_controller_->MaybeCallOnCancel(); @@ -207,15 +208,15 @@ bool ServerContextBase::CompletionOp::FinalizeResult(void** tag, bool* status) { interceptor_methods_.AddInterceptionHookPoint( ::grpc::experimental::InterceptionHookPoints::POST_RECV_CLOSE); if (interceptor_methods_.RunInterceptors()) { - /* No interceptors were run */ - if (has_tag_) { + // No interceptors were run + bool has_tag = has_tag_; + if (has_tag) { *tag = tag_; - ret = true; } Unref(); - return ret; + return has_tag; } - /* There are interceptors to be run. Return false for now */ + // There are interceptors to be run. Return false for now. return false; } From 9d82fcc1a2e456fafed728ce83e73362c0e3bdc6 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Tue, 16 Jun 2020 08:38:04 -0700 Subject: [PATCH 57/80] Fix test filter for C++ include files --- tools/run_tests/python_utils/filter_pull_request_tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/run_tests/python_utils/filter_pull_request_tests.py b/tools/run_tests/python_utils/filter_pull_request_tests.py index 98a17fd1a7c..5072df1b9bf 100644 --- a/tools/run_tests/python_utils/filter_pull_request_tests.py +++ b/tools/run_tests/python_utils/filter_pull_request_tests.py @@ -70,6 +70,7 @@ _WHITELIST_DICT = { '^doc/': [], '^examples/': [], '^include/grpc\+\+/': [_CPP_TEST_SUITE], + '^include/grpcpp/': [_CPP_TEST_SUITE], '^summerofcode/': [], '^src/cpp/': [_CPP_TEST_SUITE], '^src/csharp/': [_CSHARP_TEST_SUITE], From 67f68d7e14674bf0ff4e1bea3c3f31ced52bfc4d Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Tue, 9 Jun 2020 10:43:32 -0700 Subject: [PATCH 58/80] Add an "extras" dependency to grpcio package --- setup.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup.py b/setup.py index 2379703eb7d..b36c7cc0351 100644 --- a/setup.py +++ b/setup.py @@ -355,6 +355,9 @@ INSTALL_REQUIRES = ( "futures>=2.2.0; python_version<'3.2'", "enum34>=1.0.4; python_version<'3.4'", ) +EXTRAS_REQUIRES = { + "protobuf": "protobuf>=3.5.0.post1", +} SETUP_REQUIRES = INSTALL_REQUIRES + ( 'Sphinx~=1.8.1', @@ -417,6 +420,7 @@ setuptools.setup( package_dir=PACKAGE_DIRECTORIES, package_data=PACKAGE_DATA, install_requires=INSTALL_REQUIRES, + extras_require=EXTRAS_REQUIRES, setup_requires=SETUP_REQUIRES, cmdclass=COMMAND_CLASS, ) From d98ecdb85c2fd35b08e627cba76281cf579a548d Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Tue, 9 Jun 2020 15:46:36 -0700 Subject: [PATCH 59/80] Set version upper bound for protobuf; installs grpcio-tools instead --- requirements.bazel.txt | 2 +- requirements.txt | 2 +- setup.py | 2 +- tools/distrib/python/grpcio_tools/setup.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.bazel.txt b/requirements.bazel.txt index 0e2ecbe0907..cbe131c6846 100644 --- a/requirements.bazel.txt +++ b/requirements.bazel.txt @@ -2,7 +2,7 @@ coverage>=4.0 cython>=0.29.8 enum34>=1.0.4 -protobuf>=3.5.0.post1 +protobuf>=3.5.0.post1, < 4.0dev six>=1.10 wheel>=0.29 futures>=2.2.0 diff --git a/requirements.txt b/requirements.txt index 773e2c7c79b..f579aeda111 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ coverage>=4.0 cython>=0.29.8 enum34>=1.0.4 -protobuf>=3.5.0.post1 +protobuf>=3.5.0.post1, < 4.0dev six>=1.10 wheel>=0.29 # rsa 4.3 is the last version support Python 2 diff --git a/setup.py b/setup.py index b36c7cc0351..92c89c3bb69 100644 --- a/setup.py +++ b/setup.py @@ -356,7 +356,7 @@ INSTALL_REQUIRES = ( "enum34>=1.0.4; python_version<'3.4'", ) EXTRAS_REQUIRES = { - "protobuf": "protobuf>=3.5.0.post1", + 'grpcio-tools>={version}'.format(version=grpc_version.VERSION), } SETUP_REQUIRES = INSTALL_REQUIRES + ( diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py index 90347b529ad..9cc024f87ee 100644 --- a/tools/distrib/python/grpcio_tools/setup.py +++ b/tools/distrib/python/grpcio_tools/setup.py @@ -219,7 +219,7 @@ setuptools.setup( ext_modules=extension_modules(), packages=setuptools.find_packages('.'), install_requires=[ - 'protobuf>=3.5.0.post1', + 'protobuf>=3.5.0.post1, < 4.0dev', 'grpcio>={version}'.format(version=grpc_version.VERSION), ], package_data=package_data(), From b5ec43e182264c01123a231f526e9e992c30c2af Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Thu, 11 Jun 2020 14:08:36 -0700 Subject: [PATCH 60/80] Fix a typo in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 92c89c3bb69..807d005899d 100644 --- a/setup.py +++ b/setup.py @@ -356,7 +356,7 @@ INSTALL_REQUIRES = ( "enum34>=1.0.4; python_version<'3.4'", ) EXTRAS_REQUIRES = { - 'grpcio-tools>={version}'.format(version=grpc_version.VERSION), + 'protobuf': 'grpcio-tools>={version}'.format(version=grpc_version.VERSION), } SETUP_REQUIRES = INSTALL_REQUIRES + ( From 97904d77a40f10d6d5b69a25e8ac550944d69e09 Mon Sep 17 00:00:00 2001 From: Donna Dionne Date: Tue, 16 Jun 2020 11:03:12 -0700 Subject: [PATCH 61/80] Adding the printing of error message when unexpected status returns. --- test/cpp/end2end/xds_end2end_test.cc | 52 +++++++++++++++++++++------- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/test/cpp/end2end/xds_end2end_test.cc b/test/cpp/end2end/xds_end2end_test.cc index 688d6cfd987..9db3306738a 100644 --- a/test/cpp/end2end/xds_end2end_test.cc +++ b/test/cpp/end2end/xds_end2end_test.cc @@ -4241,9 +4241,13 @@ TEST_P(BalancerUpdateTest, UpdateBalancersButKeepUsingOriginalBalancer) { EXPECT_GT(balancers_[0]->ads_service()->eds_response_state().state, AdsServiceImpl::ResponseState::NOT_SENT); EXPECT_EQ(balancers_[1]->ads_service()->eds_response_state().state, - AdsServiceImpl::ResponseState::NOT_SENT); + AdsServiceImpl::ResponseState::NOT_SENT) + << "Error Message:" + << balancers_[1]->ads_service()->eds_response_state().error_message; EXPECT_EQ(balancers_[2]->ads_service()->eds_response_state().state, - AdsServiceImpl::ResponseState::NOT_SENT); + AdsServiceImpl::ResponseState::NOT_SENT) + << "Error Message:" + << balancers_[2]->ads_service()->eds_response_state().error_message; gpr_log(GPR_INFO, "========= ABOUT TO UPDATE 1 =========="); SetNextResolutionForLbChannel({balancers_[1]->port()}); gpr_log(GPR_INFO, "========= UPDATE 1 DONE =========="); @@ -4261,9 +4265,13 @@ TEST_P(BalancerUpdateTest, UpdateBalancersButKeepUsingOriginalBalancer) { EXPECT_GT(balancers_[0]->ads_service()->eds_response_state().state, AdsServiceImpl::ResponseState::NOT_SENT); EXPECT_EQ(balancers_[1]->ads_service()->eds_response_state().state, - AdsServiceImpl::ResponseState::NOT_SENT); + AdsServiceImpl::ResponseState::NOT_SENT) + << "Error Message:" + << balancers_[1]->ads_service()->eds_response_state().error_message; EXPECT_EQ(balancers_[2]->ads_service()->eds_response_state().state, - AdsServiceImpl::ResponseState::NOT_SENT); + AdsServiceImpl::ResponseState::NOT_SENT) + << "Error Message:" + << balancers_[2]->ads_service()->eds_response_state().error_message; } // Tests that the old LB call is still used after multiple balancer address @@ -4296,9 +4304,13 @@ TEST_P(BalancerUpdateTest, Repeated) { EXPECT_GT(balancers_[0]->ads_service()->eds_response_state().state, AdsServiceImpl::ResponseState::NOT_SENT); EXPECT_EQ(balancers_[1]->ads_service()->eds_response_state().state, - AdsServiceImpl::ResponseState::NOT_SENT); + AdsServiceImpl::ResponseState::NOT_SENT) + << "Error Message:" + << balancers_[1]->ads_service()->eds_response_state().error_message; EXPECT_EQ(balancers_[2]->ads_service()->eds_response_state().state, - AdsServiceImpl::ResponseState::NOT_SENT); + AdsServiceImpl::ResponseState::NOT_SENT) + << "Error Message:" + << balancers_[2]->ads_service()->eds_response_state().error_message; std::vector ports; ports.emplace_back(balancers_[0]->port()); ports.emplace_back(balancers_[1]->port()); @@ -4360,9 +4372,13 @@ TEST_P(BalancerUpdateTest, DeadUpdate) { EXPECT_GT(balancers_[0]->ads_service()->eds_response_state().state, AdsServiceImpl::ResponseState::NOT_SENT); EXPECT_EQ(balancers_[1]->ads_service()->eds_response_state().state, - AdsServiceImpl::ResponseState::NOT_SENT); + AdsServiceImpl::ResponseState::NOT_SENT) + << "Error Message:" + << balancers_[1]->ads_service()->eds_response_state().error_message; EXPECT_EQ(balancers_[2]->ads_service()->eds_response_state().state, - AdsServiceImpl::ResponseState::NOT_SENT); + AdsServiceImpl::ResponseState::NOT_SENT) + << "Error Message:" + << balancers_[2]->ads_service()->eds_response_state().error_message; // Kill balancer 0 gpr_log(GPR_INFO, "********** ABOUT TO KILL BALANCER 0 *************"); balancers_[0]->Shutdown(); @@ -4376,11 +4392,17 @@ TEST_P(BalancerUpdateTest, DeadUpdate) { EXPECT_EQ(0U, backends_[1]->backend_service()->request_count()); // The ADS service of no balancers sent anything EXPECT_EQ(balancers_[0]->ads_service()->eds_response_state().state, - AdsServiceImpl::ResponseState::NOT_SENT); + AdsServiceImpl::ResponseState::NOT_SENT) + << "Error Message:" + << balancers_[0]->ads_service()->eds_response_state().error_message; EXPECT_EQ(balancers_[1]->ads_service()->eds_response_state().state, - AdsServiceImpl::ResponseState::NOT_SENT); + AdsServiceImpl::ResponseState::NOT_SENT) + << "Error Message:" + << balancers_[1]->ads_service()->eds_response_state().error_message; EXPECT_EQ(balancers_[2]->ads_service()->eds_response_state().state, - AdsServiceImpl::ResponseState::NOT_SENT); + AdsServiceImpl::ResponseState::NOT_SENT) + << "Error Message:" + << balancers_[2]->ads_service()->eds_response_state().error_message; gpr_log(GPR_INFO, "========= ABOUT TO UPDATE 1 =========="); SetNextResolutionForLbChannel({balancers_[1]->port()}); gpr_log(GPR_INFO, "========= UPDATE 1 DONE =========="); @@ -4398,11 +4420,15 @@ TEST_P(BalancerUpdateTest, DeadUpdate) { EXPECT_EQ(10U, backends_[1]->backend_service()->request_count()); // The ADS service of balancer 1 sent at least 1 response. EXPECT_EQ(balancers_[0]->ads_service()->eds_response_state().state, - AdsServiceImpl::ResponseState::NOT_SENT); + AdsServiceImpl::ResponseState::NOT_SENT) + << "Error Message:" + << balancers_[0]->ads_service()->eds_response_state().error_message; EXPECT_GT(balancers_[1]->ads_service()->eds_response_state().state, AdsServiceImpl::ResponseState::NOT_SENT); EXPECT_EQ(balancers_[2]->ads_service()->eds_response_state().state, - AdsServiceImpl::ResponseState::NOT_SENT); + AdsServiceImpl::ResponseState::NOT_SENT) + << "Error Message:" + << balancers_[2]->ads_service()->eds_response_state().error_message; } // The re-resolution tests are deferred because they rely on the fallback mode, From 83e679fad1ccb6b4e3ddcb768e4d6c4f44dd8420 Mon Sep 17 00:00:00 2001 From: yulin-liang Date: Tue, 16 Jun 2020 11:16:28 -0700 Subject: [PATCH 62/80] Objc update WKT generated import paths. --- src/compiler/objective_c_plugin.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/compiler/objective_c_plugin.cc b/src/compiler/objective_c_plugin.cc index 2738a733d1f..5a5fa0e4082 100644 --- a/src/compiler/objective_c_plugin.cc +++ b/src/compiler/objective_c_plugin.cc @@ -52,15 +52,16 @@ inline ::grpc::string ImportProtoHeaders( ::grpc::string base_name = header; grpc_generator::StripPrefix(&base_name, "google/protobuf/"); + ::grpc::string file_name = "GPB" + base_name; // create the import code snippet ::grpc::string framework_header = - ::grpc::string(ProtobufLibraryFrameworkName) + "/" + base_name; + ::grpc::string(ProtobufLibraryFrameworkName) + "/" + file_name; static const ::grpc::string kFrameworkImportsCondition = "GPB_USE_PROTOBUF_FRAMEWORK_IMPORTS"; return PreprocIfElse(kFrameworkImportsCondition, indent + SystemImport(framework_header), - indent + LocalImport(header)); + indent + LocalImport(file_name)); } } // namespace From 278671072feb0d0cac32c3bd6f77fad474fb48e3 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Tue, 16 Jun 2020 12:20:19 -0700 Subject: [PATCH 63/80] Revert "Revert "Fix StartCall: make corking work and allow concurrent Start*"" --- .../impl/codegen/client_callback_impl.h | 359 +++++++++++------- .../end2end/client_callback_end2end_test.cc | 110 +++++- 2 files changed, 306 insertions(+), 163 deletions(-) diff --git a/include/grpcpp/impl/codegen/client_callback_impl.h b/include/grpcpp/impl/codegen/client_callback_impl.h index 8e683743e06..86a46d8c870 100644 --- a/include/grpcpp/impl/codegen/client_callback_impl.h +++ b/include/grpcpp/impl/codegen/client_callback_impl.h @@ -461,76 +461,51 @@ class ClientCallbackReaderWriterImpl // 1. Send initial metadata (unless corked) + recv initial metadata // 2. Any read backlog // 3. Any write backlog - // 4. Recv trailing metadata, on_completion callback - started_ = true; - - start_tag_.Set(call_.call(), - [this](bool ok) { - reactor_->OnReadInitialMetadataDone(ok); - MaybeFinish(); - }, - &start_ops_, /*can_inline=*/false); + // 4. Recv trailing metadata (unless corked) if (!start_corked_) { start_ops_.SendInitialMetadata(&context_->send_initial_metadata_, context_->initial_metadata_flags()); } - start_ops_.RecvInitialMetadata(context_); - start_ops_.set_core_cq_tag(&start_tag_); - call_.PerformOps(&start_ops_); - - // Also set up the read and write tags so that they don't have to be set up - // each time - write_tag_.Set(call_.call(), - [this](bool ok) { - reactor_->OnWriteDone(ok); - MaybeFinish(); - }, - &write_ops_, /*can_inline=*/false); - write_ops_.set_core_cq_tag(&write_tag_); - - read_tag_.Set(call_.call(), - [this](bool ok) { - reactor_->OnReadDone(ok); - MaybeFinish(); - }, - &read_ops_, /*can_inline=*/false); - read_ops_.set_core_cq_tag(&read_tag_); - if (read_ops_at_start_) { - call_.PerformOps(&read_ops_); - } - if (write_ops_at_start_) { - call_.PerformOps(&write_ops_); - } + call_.PerformOps(&start_ops_); - if (writes_done_ops_at_start_) { - call_.PerformOps(&writes_done_ops_); + { + grpc::internal::MutexLock lock(&start_mu_); + + if (backlog_.read_ops) { + call_.PerformOps(&read_ops_); + } + if (backlog_.write_ops) { + call_.PerformOps(&write_ops_); + } + if (backlog_.writes_done_ops) { + call_.PerformOps(&writes_done_ops_); + } + call_.PerformOps(&finish_ops_); + // The last thing in this critical section is to set started_ so that it + // can be used lock-free as well. + started_.store(true, std::memory_order_release); } - - finish_tag_.Set(call_.call(), [this](bool /*ok*/) { MaybeFinish(); }, - &finish_ops_, /*can_inline=*/false); - finish_ops_.ClientRecvStatus(context_, &finish_status_); - finish_ops_.set_core_cq_tag(&finish_tag_); - call_.PerformOps(&finish_ops_); + // MaybeFinish outside the lock to make sure that destruction of this object + // doesn't take place while holding the lock (which would cause the lock to + // be released after destruction) + this->MaybeFinish(); } void Read(Response* msg) override { read_ops_.RecvMessage(msg); callbacks_outstanding_.fetch_add(1, std::memory_order_relaxed); - if (started_) { - call_.PerformOps(&read_ops_); - } else { - read_ops_at_start_ = true; + if (GPR_UNLIKELY(!started_.load(std::memory_order_acquire))) { + grpc::internal::MutexLock lock(&start_mu_); + if (GPR_LIKELY(!started_.load(std::memory_order_relaxed))) { + backlog_.read_ops = true; + return; + } } + call_.PerformOps(&read_ops_); } void Write(const Request* msg, ::grpc::WriteOptions options) override { - if (start_corked_) { - write_ops_.SendInitialMetadata(&context_->send_initial_metadata_, - context_->initial_metadata_flags()); - start_corked_ = false; - } - if (options.is_last_message()) { options.set_buffer_hint(); write_ops_.ClientSendClose(); @@ -538,18 +513,22 @@ class ClientCallbackReaderWriterImpl // TODO(vjpai): don't assert GPR_CODEGEN_ASSERT(write_ops_.SendMessagePtr(msg, options).ok()); callbacks_outstanding_.fetch_add(1, std::memory_order_relaxed); - if (started_) { - call_.PerformOps(&write_ops_); - } else { - write_ops_at_start_ = true; + if (GPR_UNLIKELY(corked_write_needed_)) { + write_ops_.SendInitialMetadata(&context_->send_initial_metadata_, + context_->initial_metadata_flags()); + corked_write_needed_ = false; + } + + if (GPR_UNLIKELY(!started_.load(std::memory_order_acquire))) { + grpc::internal::MutexLock lock(&start_mu_); + if (GPR_LIKELY(!started_.load(std::memory_order_relaxed))) { + backlog_.write_ops = true; + return; + } } + call_.PerformOps(&write_ops_); } void WritesDone() override { - if (start_corked_) { - writes_done_ops_.SendInitialMetadata(&context_->send_initial_metadata_, - context_->initial_metadata_flags()); - start_corked_ = false; - } writes_done_ops_.ClientSendClose(); writes_done_tag_.Set(call_.call(), [this](bool ok) { @@ -559,11 +538,19 @@ class ClientCallbackReaderWriterImpl &writes_done_ops_, /*can_inline=*/false); writes_done_ops_.set_core_cq_tag(&writes_done_tag_); callbacks_outstanding_.fetch_add(1, std::memory_order_relaxed); - if (started_) { - call_.PerformOps(&writes_done_ops_); - } else { - writes_done_ops_at_start_ = true; + if (GPR_UNLIKELY(corked_write_needed_)) { + writes_done_ops_.SendInitialMetadata(&context_->send_initial_metadata_, + context_->initial_metadata_flags()); + corked_write_needed_ = false; } + if (GPR_UNLIKELY(!started_.load(std::memory_order_acquire))) { + grpc::internal::MutexLock lock(&start_mu_); + if (GPR_LIKELY(!started_.load(std::memory_order_relaxed))) { + backlog_.writes_done_ops = true; + return; + } + } + call_.PerformOps(&writes_done_ops_); } void AddHold(int holds) override { @@ -580,8 +567,42 @@ class ClientCallbackReaderWriterImpl : context_(context), call_(call), reactor_(reactor), - start_corked_(context_->initial_metadata_corked_) { + start_corked_(context_->initial_metadata_corked_), + corked_write_needed_(start_corked_) { this->BindReactor(reactor); + + // Set up the unchanging parts of the start, read, and write tags and ops. + start_tag_.Set(call_.call(), + [this](bool ok) { + reactor_->OnReadInitialMetadataDone(ok); + MaybeFinish(); + }, + &start_ops_, /*can_inline=*/false); + start_ops_.RecvInitialMetadata(context_); + start_ops_.set_core_cq_tag(&start_tag_); + + write_tag_.Set(call_.call(), + [this](bool ok) { + reactor_->OnWriteDone(ok); + MaybeFinish(); + }, + &write_ops_, /*can_inline=*/false); + write_ops_.set_core_cq_tag(&write_tag_); + + read_tag_.Set(call_.call(), + [this](bool ok) { + reactor_->OnReadDone(ok); + MaybeFinish(); + }, + &read_ops_, /*can_inline=*/false); + read_ops_.set_core_cq_tag(&read_tag_); + + // Also set up the Finish tag and op set. + finish_tag_.Set(call_.call(), [this](bool /*ok*/) { MaybeFinish(); }, + &finish_ops_, + /*can_inline=*/false); + finish_ops_.ClientRecvStatus(context_, &finish_status_); + finish_ops_.set_core_cq_tag(&finish_tag_); } ::grpc_impl::ClientContext* const context_; @@ -592,7 +613,9 @@ class ClientCallbackReaderWriterImpl grpc::internal::CallOpRecvInitialMetadata> start_ops_; grpc::internal::CallbackWithSuccessTag start_tag_; - bool start_corked_; + const bool start_corked_; + bool corked_write_needed_; // no lock needed since only accessed in + // Write/WritesDone which cannot be concurrent grpc::internal::CallOpSet finish_ops_; grpc::internal::CallbackWithSuccessTag finish_tag_; @@ -603,22 +626,27 @@ class ClientCallbackReaderWriterImpl grpc::internal::CallOpClientSendClose> write_ops_; grpc::internal::CallbackWithSuccessTag write_tag_; - bool write_ops_at_start_{false}; grpc::internal::CallOpSet writes_done_ops_; grpc::internal::CallbackWithSuccessTag writes_done_tag_; - bool writes_done_ops_at_start_{false}; grpc::internal::CallOpSet> read_ops_; grpc::internal::CallbackWithSuccessTag read_tag_; - bool read_ops_at_start_{false}; - // Minimum of 2 callbacks to pre-register for start and finish - std::atomic callbacks_outstanding_{2}; - bool started_{false}; + struct StartCallBacklog { + bool write_ops = false; + bool writes_done_ops = false; + bool read_ops = false; + }; + StartCallBacklog backlog_ /* GUARDED_BY(start_mu_) */; + + // Minimum of 3 callbacks to pre-register for start ops, StartCall, and finish + std::atomic callbacks_outstanding_{3}; + std::atomic_bool started_{false}; + grpc::internal::Mutex start_mu_; }; template @@ -670,8 +698,7 @@ class ClientCallbackReaderImpl : public ClientCallbackReader { // This call initiates two batches, plus any backlog, each with a callback // 1. Send initial metadata (unless corked) + recv initial metadata // 2. Any backlog - // 3. Recv trailing metadata, on_completion callback - started_ = true; + // 3. Recv trailing metadata start_tag_.Set(call_.call(), [this](bool ok) { @@ -693,8 +720,13 @@ class ClientCallbackReaderImpl : public ClientCallbackReader { }, &read_ops_, /*can_inline=*/false); read_ops_.set_core_cq_tag(&read_tag_); - if (read_ops_at_start_) { - call_.PerformOps(&read_ops_); + + { + grpc::internal::MutexLock lock(&start_mu_); + if (backlog_.read_ops) { + call_.PerformOps(&read_ops_); + } + started_.store(true, std::memory_order_release); } finish_tag_.Set(call_.call(), [this](bool /*ok*/) { MaybeFinish(); }, @@ -707,11 +739,14 @@ class ClientCallbackReaderImpl : public ClientCallbackReader { void Read(Response* msg) override { read_ops_.RecvMessage(msg); callbacks_outstanding_.fetch_add(1, std::memory_order_relaxed); - if (started_) { - call_.PerformOps(&read_ops_); - } else { - read_ops_at_start_ = true; + if (GPR_UNLIKELY(!started_.load(std::memory_order_acquire))) { + grpc::internal::MutexLock lock(&start_mu_); + if (GPR_LIKELY(!started_.load(std::memory_order_relaxed))) { + backlog_.read_ops = true; + return; + } } + call_.PerformOps(&read_ops_); } void AddHold(int holds) override { @@ -752,11 +787,16 @@ class ClientCallbackReaderImpl : public ClientCallbackReader { grpc::internal::CallOpSet> read_ops_; grpc::internal::CallbackWithSuccessTag read_tag_; - bool read_ops_at_start_{false}; + + struct StartCallBacklog { + bool read_ops = false; + }; + StartCallBacklog backlog_ /* GUARDED_BY(start_mu_) */; // Minimum of 2 callbacks to pre-register for start and finish std::atomic callbacks_outstanding_{2}; - bool started_{false}; + std::atomic_bool started_{false}; + grpc::internal::Mutex start_mu_; }; template @@ -809,74 +849,60 @@ class ClientCallbackWriterImpl : public ClientCallbackWriter { // This call initiates two batches, plus any backlog, each with a callback // 1. Send initial metadata (unless corked) + recv initial metadata // 2. Any backlog - // 3. Recv trailing metadata, on_completion callback - started_ = true; + // 3. Recv trailing metadata - start_tag_.Set(call_.call(), - [this](bool ok) { - reactor_->OnReadInitialMetadataDone(ok); - MaybeFinish(); - }, - &start_ops_, /*can_inline=*/false); if (!start_corked_) { start_ops_.SendInitialMetadata(&context_->send_initial_metadata_, context_->initial_metadata_flags()); } - start_ops_.RecvInitialMetadata(context_); - start_ops_.set_core_cq_tag(&start_tag_); call_.PerformOps(&start_ops_); - // Also set up the read and write tags so that they don't have to be set up - // each time - write_tag_.Set(call_.call(), - [this](bool ok) { - reactor_->OnWriteDone(ok); - MaybeFinish(); - }, - &write_ops_, /*can_inline=*/false); - write_ops_.set_core_cq_tag(&write_tag_); - - if (write_ops_at_start_) { - call_.PerformOps(&write_ops_); + { + grpc::internal::MutexLock lock(&start_mu_); + + if (backlog_.write_ops) { + call_.PerformOps(&write_ops_); + } + if (backlog_.writes_done_ops) { + call_.PerformOps(&writes_done_ops_); + } + call_.PerformOps(&finish_ops_); + // The last thing in this critical section is to set started_ so that it + // can be used lock-free as well. + started_.store(true, std::memory_order_release); } - - if (writes_done_ops_at_start_) { - call_.PerformOps(&writes_done_ops_); - } - - finish_tag_.Set(call_.call(), [this](bool /*ok*/) { MaybeFinish(); }, - &finish_ops_, /*can_inline=*/false); - finish_ops_.ClientRecvStatus(context_, &finish_status_); - finish_ops_.set_core_cq_tag(&finish_tag_); - call_.PerformOps(&finish_ops_); + // MaybeFinish outside the lock to make sure that destruction of this object + // doesn't take place while holding the lock (which would cause the lock to + // be released after destruction) + this->MaybeFinish(); } void Write(const Request* msg, ::grpc::WriteOptions options) override { - if (start_corked_) { - write_ops_.SendInitialMetadata(&context_->send_initial_metadata_, - context_->initial_metadata_flags()); - start_corked_ = false; - } - - if (options.is_last_message()) { + if (GPR_UNLIKELY(options.is_last_message())) { options.set_buffer_hint(); write_ops_.ClientSendClose(); } // TODO(vjpai): don't assert GPR_CODEGEN_ASSERT(write_ops_.SendMessagePtr(msg, options).ok()); callbacks_outstanding_.fetch_add(1, std::memory_order_relaxed); - if (started_) { - call_.PerformOps(&write_ops_); - } else { - write_ops_at_start_ = true; + + if (GPR_UNLIKELY(corked_write_needed_)) { + write_ops_.SendInitialMetadata(&context_->send_initial_metadata_, + context_->initial_metadata_flags()); + corked_write_needed_ = false; } + + if (GPR_UNLIKELY(!started_.load(std::memory_order_acquire))) { + grpc::internal::MutexLock lock(&start_mu_); + if (GPR_LIKELY(!started_.load(std::memory_order_relaxed))) { + backlog_.write_ops = true; + return; + } + } + call_.PerformOps(&write_ops_); } + void WritesDone() override { - if (start_corked_) { - writes_done_ops_.SendInitialMetadata(&context_->send_initial_metadata_, - context_->initial_metadata_flags()); - start_corked_ = false; - } writes_done_ops_.ClientSendClose(); writes_done_tag_.Set(call_.call(), [this](bool ok) { @@ -886,11 +912,21 @@ class ClientCallbackWriterImpl : public ClientCallbackWriter { &writes_done_ops_, /*can_inline=*/false); writes_done_ops_.set_core_cq_tag(&writes_done_tag_); callbacks_outstanding_.fetch_add(1, std::memory_order_relaxed); - if (started_) { - call_.PerformOps(&writes_done_ops_); - } else { - writes_done_ops_at_start_ = true; + + if (GPR_UNLIKELY(corked_write_needed_)) { + writes_done_ops_.SendInitialMetadata(&context_->send_initial_metadata_, + context_->initial_metadata_flags()); + corked_write_needed_ = false; + } + + if (GPR_UNLIKELY(!started_.load(std::memory_order_acquire))) { + grpc::internal::MutexLock lock(&start_mu_); + if (GPR_LIKELY(!started_.load(std::memory_order_relaxed))) { + backlog_.writes_done_ops = true; + return; + } } + call_.PerformOps(&writes_done_ops_); } void AddHold(int holds) override { @@ -909,10 +945,36 @@ class ClientCallbackWriterImpl : public ClientCallbackWriter { : context_(context), call_(call), reactor_(reactor), - start_corked_(context_->initial_metadata_corked_) { + start_corked_(context_->initial_metadata_corked_), + corked_write_needed_(start_corked_) { this->BindReactor(reactor); + + // Set up the unchanging parts of the start and write tags and ops. + start_tag_.Set(call_.call(), + [this](bool ok) { + reactor_->OnReadInitialMetadataDone(ok); + MaybeFinish(); + }, + &start_ops_, /*can_inline=*/false); + start_ops_.RecvInitialMetadata(context_); + start_ops_.set_core_cq_tag(&start_tag_); + + write_tag_.Set(call_.call(), + [this](bool ok) { + reactor_->OnWriteDone(ok); + MaybeFinish(); + }, + &write_ops_, /*can_inline=*/false); + write_ops_.set_core_cq_tag(&write_tag_); + + // Also set up the Finish tag and op set. finish_ops_.RecvMessage(response); finish_ops_.AllowNoMessage(); + finish_tag_.Set(call_.call(), [this](bool /*ok*/) { MaybeFinish(); }, + &finish_ops_, + /*can_inline=*/false); + finish_ops_.ClientRecvStatus(context_, &finish_status_); + finish_ops_.set_core_cq_tag(&finish_tag_); } ::grpc_impl::ClientContext* const context_; @@ -923,7 +985,9 @@ class ClientCallbackWriterImpl : public ClientCallbackWriter { grpc::internal::CallOpRecvInitialMetadata> start_ops_; grpc::internal::CallbackWithSuccessTag start_tag_; - bool start_corked_; + const bool start_corked_; + bool corked_write_needed_; // no lock needed since only accessed in + // Write/WritesDone which cannot be concurrent grpc::internal::CallOpSet @@ -936,17 +1000,22 @@ class ClientCallbackWriterImpl : public ClientCallbackWriter { grpc::internal::CallOpClientSendClose> write_ops_; grpc::internal::CallbackWithSuccessTag write_tag_; - bool write_ops_at_start_{false}; grpc::internal::CallOpSet writes_done_ops_; grpc::internal::CallbackWithSuccessTag writes_done_tag_; - bool writes_done_ops_at_start_{false}; - // Minimum of 2 callbacks to pre-register for start and finish - std::atomic callbacks_outstanding_{2}; - bool started_{false}; + struct StartCallBacklog { + bool write_ops = false; + bool writes_done_ops = false; + }; + StartCallBacklog backlog_ /* GUARDED_BY(start_mu_) */; + + // Minimum of 3 callbacks to pre-register for start ops, StartCall, and finish + std::atomic callbacks_outstanding_{3}; + std::atomic_bool started_{false}; + grpc::internal::Mutex start_mu_; }; template @@ -985,7 +1054,6 @@ class ClientCallbackUnaryImpl final : public ClientCallbackUnary { // This call initiates two batches, each with a callback // 1. Send initial metadata + write + writes done + recv initial metadata // 2. Read message, recv trailing metadata - started_ = true; start_tag_.Set(call_.call(), [this](bool ok) { @@ -1053,7 +1121,6 @@ class ClientCallbackUnaryImpl final : public ClientCallbackUnary { // This call will have 2 callbacks: start and finish std::atomic callbacks_outstanding_{2}; - bool started_{false}; }; class ClientCallbackUnaryFactory { diff --git a/test/cpp/end2end/client_callback_end2end_test.cc b/test/cpp/end2end/client_callback_end2end_test.cc index 05aac336789..cd3c14881bd 100644 --- a/test/cpp/end2end/client_callback_end2end_test.cc +++ b/test/cpp/end2end/client_callback_end2end_test.cc @@ -16,12 +16,6 @@ * */ -#include -#include -#include -#include -#include - #include #include #include @@ -31,6 +25,14 @@ #include #include #include +#include + +#include +#include +#include +#include +#include +#include #include "src/core/lib/gpr/env.h" #include "src/core/lib/iomgr/iomgr.h" @@ -43,8 +45,6 @@ #include "test/cpp/util/string_ref_helper.h" #include "test/cpp/util/test_credentials_provider.h" -#include - // MAYBE_SKIP_TEST is a macro to determine if this particular test configuration // should be skipped based on a decision made at SetUp time. In particular, any // callback tests can only be run if the iomgr can run in the background or if @@ -1114,7 +1114,8 @@ class BidiClient public: BidiClient(grpc::testing::EchoTestService::Stub* stub, ServerTryCancelRequestPhase server_try_cancel, - int num_msgs_to_send, ClientCancelInfo client_cancel = {}) + int num_msgs_to_send, bool cork_metadata, bool first_write_async, + ClientCancelInfo client_cancel = {}) : server_try_cancel_(server_try_cancel), msgs_to_send_{num_msgs_to_send}, client_cancel_{client_cancel} { @@ -1124,8 +1125,9 @@ class BidiClient grpc::to_string(server_try_cancel)); } request_.set_message("Hello fren "); + context_.set_initial_metadata_corked(cork_metadata); stub->experimental_async()->BidiStream(&context_, this); - MaybeWrite(); + MaybeAsyncWrite(first_write_async); StartRead(&response_); StartCall(); } @@ -1146,6 +1148,10 @@ class BidiClient } } void OnWriteDone(bool ok) override { + if (async_write_thread_.joinable()) { + async_write_thread_.join(); + RemoveHold(); + } if (server_try_cancel_ == DO_NOT_CANCEL) { EXPECT_TRUE(ok); } else if (!ok) { @@ -1210,6 +1216,26 @@ class BidiClient } private: + void MaybeAsyncWrite(bool first_write_async) { + if (first_write_async) { + // Make sure that we have a write to issue. + // TODO(vjpai): Make this work with 0 writes case as well. + assert(msgs_to_send_ >= 1); + + AddHold(); + async_write_thread_ = std::thread([this] { + std::unique_lock lock(async_write_thread_mu_); + async_write_thread_cv_.wait( + lock, [this] { return async_write_thread_start_; }); + MaybeWrite(); + }); + std::lock_guard lock(async_write_thread_mu_); + async_write_thread_start_ = true; + async_write_thread_cv_.notify_one(); + return; + } + MaybeWrite(); + } void MaybeWrite() { if (client_cancel_.cancel && writes_complete_ == client_cancel_.ops_before_cancel) { @@ -1231,13 +1257,57 @@ class BidiClient std::mutex mu_; std::condition_variable cv_; bool done_ = false; + std::thread async_write_thread_; + bool async_write_thread_start_ = false; + std::mutex async_write_thread_mu_; + std::condition_variable async_write_thread_cv_; }; TEST_P(ClientCallbackEnd2endTest, BidiStream) { MAYBE_SKIP_TEST; ResetStub(); - BidiClient test{stub_.get(), DO_NOT_CANCEL, - kServerDefaultResponseStreamsToSend}; + BidiClient test(stub_.get(), DO_NOT_CANCEL, + kServerDefaultResponseStreamsToSend, + /*cork_metadata=*/false, /*first_write_async=*/false); + test.Await(); + // Make sure that the server interceptors were not notified of a cancel + if (GetParam().use_interceptors) { + EXPECT_EQ(0, DummyInterceptor::GetNumTimesCancel()); + } +} + +TEST_P(ClientCallbackEnd2endTest, BidiStreamFirstWriteAsync) { + MAYBE_SKIP_TEST; + ResetStub(); + BidiClient test(stub_.get(), DO_NOT_CANCEL, + kServerDefaultResponseStreamsToSend, + /*cork_metadata=*/false, /*first_write_async=*/true); + test.Await(); + // Make sure that the server interceptors were not notified of a cancel + if (GetParam().use_interceptors) { + EXPECT_EQ(0, DummyInterceptor::GetNumTimesCancel()); + } +} + +TEST_P(ClientCallbackEnd2endTest, BidiStreamCorked) { + MAYBE_SKIP_TEST; + ResetStub(); + BidiClient test(stub_.get(), DO_NOT_CANCEL, + kServerDefaultResponseStreamsToSend, + /*cork_metadata=*/true, /*first_write_async=*/false); + test.Await(); + // Make sure that the server interceptors were not notified of a cancel + if (GetParam().use_interceptors) { + EXPECT_EQ(0, DummyInterceptor::GetNumTimesCancel()); + } +} + +TEST_P(ClientCallbackEnd2endTest, BidiStreamCorkedFirstWriteAsync) { + MAYBE_SKIP_TEST; + ResetStub(); + BidiClient test(stub_.get(), DO_NOT_CANCEL, + kServerDefaultResponseStreamsToSend, + /*cork_metadata=*/true, /*first_write_async=*/true); test.Await(); // Make sure that the server interceptors were not notified of a cancel if (GetParam().use_interceptors) { @@ -1248,8 +1318,10 @@ TEST_P(ClientCallbackEnd2endTest, BidiStream) { TEST_P(ClientCallbackEnd2endTest, ClientCancelsBidiStream) { MAYBE_SKIP_TEST; ResetStub(); - BidiClient test{stub_.get(), DO_NOT_CANCEL, - kServerDefaultResponseStreamsToSend, ClientCancelInfo{2}}; + BidiClient test(stub_.get(), DO_NOT_CANCEL, + kServerDefaultResponseStreamsToSend, + /*cork_metadata=*/false, /*first_write_async=*/false, + ClientCancelInfo(2)); test.Await(); // Make sure that the server interceptors were notified of a cancel if (GetParam().use_interceptors) { @@ -1261,7 +1333,8 @@ TEST_P(ClientCallbackEnd2endTest, ClientCancelsBidiStream) { TEST_P(ClientCallbackEnd2endTest, BidiStreamServerCancelBefore) { MAYBE_SKIP_TEST; ResetStub(); - BidiClient test{stub_.get(), CANCEL_BEFORE_PROCESSING, 2}; + BidiClient test(stub_.get(), CANCEL_BEFORE_PROCESSING, /*num_msgs_to_send=*/2, + /*cork_metadata=*/false, /*first_write_async=*/false); test.Await(); // Make sure that the server interceptors were notified if (GetParam().use_interceptors) { @@ -1274,7 +1347,9 @@ TEST_P(ClientCallbackEnd2endTest, BidiStreamServerCancelBefore) { TEST_P(ClientCallbackEnd2endTest, BidiStreamServerCancelDuring) { MAYBE_SKIP_TEST; ResetStub(); - BidiClient test{stub_.get(), CANCEL_DURING_PROCESSING, 10}; + BidiClient test(stub_.get(), CANCEL_DURING_PROCESSING, + /*num_msgs_to_send=*/10, /*cork_metadata=*/false, + /*first_write_async=*/false); test.Await(); // Make sure that the server interceptors were notified if (GetParam().use_interceptors) { @@ -1287,7 +1362,8 @@ TEST_P(ClientCallbackEnd2endTest, BidiStreamServerCancelDuring) { TEST_P(ClientCallbackEnd2endTest, BidiStreamServerCancelAfter) { MAYBE_SKIP_TEST; ResetStub(); - BidiClient test{stub_.get(), CANCEL_AFTER_PROCESSING, 5}; + BidiClient test(stub_.get(), CANCEL_AFTER_PROCESSING, /*num_msgs_to_send=*/5, + /*cork_metadata=*/false, /*first_write_async=*/false); test.Await(); // Make sure that the server interceptors were notified if (GetParam().use_interceptors) { From 7eebba37817d16ba7baeb9c2fb94984c25dec695 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Tue, 16 Jun 2020 12:30:01 -0700 Subject: [PATCH 64/80] Fixes needed to make roll-forward of StartCall and corking work --- BUILD | 1 + BUILD.gn | 1 + CMakeLists.txt | 2 + Makefile | 2 + build_autogenerated.yaml | 2 + gRPC-C++.podspec | 1 + grpc.gyp | 2 + .../impl/codegen/client_callback_impl.h | 128 +++++++++++++----- src/cpp/client/client_callback.cc | 52 +++++++ .../end2end/client_callback_end2end_test.cc | 8 +- tools/doxygen/Doxyfile.c++.internal | 1 + 11 files changed, 162 insertions(+), 38 deletions(-) create mode 100644 src/cpp/client/client_callback.cc diff --git a/BUILD b/BUILD index 0d2a7ca058c..99750d060cc 100644 --- a/BUILD +++ b/BUILD @@ -124,6 +124,7 @@ GRPC_SECURE_PUBLIC_HDRS = [ # TODO(ctiller): layer grpc atop grpc_unsecure, layer grpc++ atop grpc++_unsecure GRPCXX_SRCS = [ "src/cpp/client/channel_cc.cc", + "src/cpp/client/client_callback.cc", "src/cpp/client/client_context.cc", "src/cpp/client/client_interceptor.cc", "src/cpp/client/create_channel.cc", diff --git a/BUILD.gn b/BUILD.gn index a16d298f1be..378250dd2c0 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -1220,6 +1220,7 @@ config("grpc_config") { "include/grpcpp/support/time.h", "include/grpcpp/support/validate_service_config.h", "src/cpp/client/channel_cc.cc", + "src/cpp/client/client_callback.cc", "src/cpp/client/client_context.cc", "src/cpp/client/client_interceptor.cc", "src/cpp/client/create_channel.cc", diff --git a/CMakeLists.txt b/CMakeLists.txt index 158b3c03f34..4e0c695a5bc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2482,6 +2482,7 @@ endif() add_library(grpc++ src/cpp/client/channel_cc.cc + src/cpp/client/client_callback.cc src/cpp/client/client_context.cc src/cpp/client/client_interceptor.cc src/cpp/client/create_channel.cc @@ -3183,6 +3184,7 @@ endif() add_library(grpc++_unsecure src/cpp/client/channel_cc.cc + src/cpp/client/client_callback.cc src/cpp/client/client_context.cc src/cpp/client/client_interceptor.cc src/cpp/client/create_channel.cc diff --git a/Makefile b/Makefile index 80d3be9e53a..83c7600a698 100644 --- a/Makefile +++ b/Makefile @@ -4717,6 +4717,7 @@ $(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/helpers.o: $(GENDIR)/src/proto/grpc LIBGRPC++_SRC = \ src/cpp/client/channel_cc.cc \ + src/cpp/client/client_callback.cc \ src/cpp/client/client_context.cc \ src/cpp/client/client_interceptor.cc \ src/cpp/client/create_channel.cc \ @@ -5423,6 +5424,7 @@ endif LIBGRPC++_UNSECURE_SRC = \ src/cpp/client/channel_cc.cc \ + src/cpp/client/client_callback.cc \ src/cpp/client/client_context.cc \ src/cpp/client/client_interceptor.cc \ src/cpp/client/create_channel.cc \ diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index 9eb53b8e576..5f09c7b3e73 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -2203,6 +2203,7 @@ libs: - src/cpp/thread_manager/thread_manager.h src: - src/cpp/client/channel_cc.cc + - src/cpp/client/client_callback.cc - src/cpp/client/client_context.cc - src/cpp/client/client_interceptor.cc - src/cpp/client/create_channel.cc @@ -2592,6 +2593,7 @@ libs: - src/cpp/thread_manager/thread_manager.h src: - src/cpp/client/channel_cc.cc + - src/cpp/client/client_callback.cc - src/cpp/client/client_context.cc - src/cpp/client/client_interceptor.cc - src/cpp/client/create_channel.cc diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index d8f404abc1f..8851119c07f 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -616,6 +616,7 @@ Pod::Spec.new do |s| 'src/core/tsi/transport_security_grpc.h', 'src/core/tsi/transport_security_interface.h', 'src/cpp/client/channel_cc.cc', + 'src/cpp/client/client_callback.cc', 'src/cpp/client/client_context.cc', 'src/cpp/client/client_interceptor.cc', 'src/cpp/client/create_channel.cc', diff --git a/grpc.gyp b/grpc.gyp index b80efd83534..4022227f5e3 100644 --- a/grpc.gyp +++ b/grpc.gyp @@ -1313,6 +1313,7 @@ ], 'sources': [ 'src/cpp/client/channel_cc.cc', + 'src/cpp/client/client_callback.cc', 'src/cpp/client/client_context.cc', 'src/cpp/client/client_interceptor.cc', 'src/cpp/client/create_channel.cc', @@ -1464,6 +1465,7 @@ ], 'sources': [ 'src/cpp/client/channel_cc.cc', + 'src/cpp/client/client_callback.cc', 'src/cpp/client/client_context.cc', 'src/cpp/client/client_interceptor.cc', 'src/cpp/client/create_channel.cc', diff --git a/include/grpcpp/impl/codegen/client_callback_impl.h b/include/grpcpp/impl/codegen/client_callback_impl.h index 86a46d8c870..3ac46b788ea 100644 --- a/include/grpcpp/impl/codegen/client_callback_impl.h +++ b/include/grpcpp/impl/codegen/client_callback_impl.h @@ -101,6 +101,29 @@ class CallbackUnaryCallImpl { call.PerformOps(ops); } }; + +// Base class for public API classes. +class ClientReactor { + public: + /// Called by the library when all operations associated with this RPC have + /// completed and all Holds have been removed. OnDone provides the RPC status + /// outcome for both successful and failed RPCs. If it is never called on an + /// RPC, it indicates an application-level problem (like failure to remove a + /// hold). + /// + /// \param[in] s The status outcome of this RPC + virtual void OnDone(const ::grpc::Status& /*s*/) = 0; + + /// InternalScheduleOnDone is not part of the API and is not meant to be + /// overridden. It is virtual to allow successful builds for certain bazel + /// build users that only want to depend on gRPC codegen headers and not the + /// full library (although this is not a generally-supported option). Although + /// the virtual call is slower than a direct call, this function is + /// heavyweight and the cost of the virtual call is not much in comparison. + /// This function may be removed or devirtualized in the future. + virtual void InternalScheduleOnDone(::grpc::Status s); +}; + } // namespace internal // Forward declarations @@ -189,7 +212,7 @@ class ClientCallbackUnary { /// \a ClientBidiReactor is the interface for a bidirectional streaming RPC. template -class ClientBidiReactor { +class ClientBidiReactor : public internal::ClientReactor { public: virtual ~ClientBidiReactor() {} @@ -282,7 +305,7 @@ class ClientBidiReactor { /// (like failure to remove a hold). /// /// \param[in] s The status outcome of this RPC - virtual void OnDone(const ::grpc::Status& /*s*/) {} + void OnDone(const ::grpc::Status& /*s*/) override {} /// Notifies the application that a read of initial metadata from the /// server is done. If the application chooses not to implement this method, @@ -327,7 +350,7 @@ class ClientBidiReactor { /// \a ClientReadReactor is the interface for a server-streaming RPC. /// All public methods behave as in ClientBidiReactor. template -class ClientReadReactor { +class ClientReadReactor : public internal::ClientReactor { public: virtual ~ClientReadReactor() {} @@ -341,7 +364,7 @@ class ClientReadReactor { } void RemoveHold() { reader_->RemoveHold(); } - virtual void OnDone(const ::grpc::Status& /*s*/) {} + void OnDone(const ::grpc::Status& /*s*/) override {} virtual void OnReadInitialMetadataDone(bool /*ok*/) {} virtual void OnReadDone(bool /*ok*/) {} @@ -354,7 +377,7 @@ class ClientReadReactor { /// \a ClientWriteReactor is the interface for a client-streaming RPC. /// All public methods behave as in ClientBidiReactor. template -class ClientWriteReactor { +class ClientWriteReactor : public internal::ClientReactor { public: virtual ~ClientWriteReactor() {} @@ -377,7 +400,7 @@ class ClientWriteReactor { } void RemoveHold() { writer_->RemoveHold(); } - virtual void OnDone(const ::grpc::Status& /*s*/) {} + void OnDone(const ::grpc::Status& /*s*/) override {} virtual void OnReadInitialMetadataDone(bool /*ok*/) {} virtual void OnWriteDone(bool /*ok*/) {} virtual void OnWritesDoneDone(bool /*ok*/) {} @@ -385,6 +408,7 @@ class ClientWriteReactor { private: friend class ClientCallbackWriter; void BindWriter(ClientCallbackWriter* writer) { writer_ = writer; } + ClientCallbackWriter* writer_; }; @@ -399,12 +423,12 @@ class ClientWriteReactor { /// call (that is part of the unary call itself) and there is no reactor object /// being created as a result of this call, we keep a consistent 2-phase /// initiation API among all the reactor flavors. -class ClientUnaryReactor { +class ClientUnaryReactor : public internal::ClientReactor { public: virtual ~ClientUnaryReactor() {} void StartCall() { call_->StartCall(); } - virtual void OnDone(const ::grpc::Status& /*s*/) {} + void OnDone(const ::grpc::Status& /*s*/) override {} virtual void OnReadInitialMetadataDone(bool /*ok*/) {} private: @@ -444,7 +468,13 @@ class ClientCallbackReaderWriterImpl // there are no tests catching the compiler warning. static void operator delete(void*, void*) { GPR_CODEGEN_ASSERT(false); } - void MaybeFinish() { + // MaybeFinish can be called from reactions or from user-initiated operations + // like StartCall or RemoveHold. If this is the last operation or hold on this + // object, it will invoke the OnDone reaction. If MaybeFinish was called from + // a reaction, it can call OnDone directly. If not, it would need to schedule + // OnDone onto an executor thread to avoid the possibility of deadlocking with + // any locks in the user code that invoked it. + void MaybeFinish(bool from_reaction) { if (GPR_UNLIKELY(callbacks_outstanding_.fetch_sub( 1, std::memory_order_acq_rel) == 1)) { ::grpc::Status s = std::move(finish_status_); @@ -452,7 +482,11 @@ class ClientCallbackReaderWriterImpl auto* call = call_.call(); this->~ClientCallbackReaderWriterImpl(); ::grpc::g_core_codegen_interface->grpc_call_unref(call); - reactor->OnDone(s); + if (GPR_LIKELY(from_reaction)) { + reactor->OnDone(s); + } else { + reactor->InternalScheduleOnDone(std::move(s)); + } } } @@ -489,7 +523,7 @@ class ClientCallbackReaderWriterImpl // MaybeFinish outside the lock to make sure that destruction of this object // doesn't take place while holding the lock (which would cause the lock to // be released after destruction) - this->MaybeFinish(); + this->MaybeFinish(/*from_reaction=*/false); } void Read(Response* msg) override { @@ -533,7 +567,7 @@ class ClientCallbackReaderWriterImpl writes_done_tag_.Set(call_.call(), [this](bool ok) { reactor_->OnWritesDoneDone(ok); - MaybeFinish(); + MaybeFinish(/*from_reaction=*/true); }, &writes_done_ops_, /*can_inline=*/false); writes_done_ops_.set_core_cq_tag(&writes_done_tag_); @@ -556,7 +590,7 @@ class ClientCallbackReaderWriterImpl void AddHold(int holds) override { callbacks_outstanding_.fetch_add(holds, std::memory_order_relaxed); } - void RemoveHold() override { MaybeFinish(); } + void RemoveHold() override { MaybeFinish(/*from_reaction=*/false); } private: friend class ClientCallbackReaderWriterFactory; @@ -575,7 +609,7 @@ class ClientCallbackReaderWriterImpl start_tag_.Set(call_.call(), [this](bool ok) { reactor_->OnReadInitialMetadataDone(ok); - MaybeFinish(); + MaybeFinish(/*from_reaction=*/true); }, &start_ops_, /*can_inline=*/false); start_ops_.RecvInitialMetadata(context_); @@ -584,7 +618,7 @@ class ClientCallbackReaderWriterImpl write_tag_.Set(call_.call(), [this](bool ok) { reactor_->OnWriteDone(ok); - MaybeFinish(); + MaybeFinish(/*from_reaction=*/true); }, &write_ops_, /*can_inline=*/false); write_ops_.set_core_cq_tag(&write_tag_); @@ -592,15 +626,17 @@ class ClientCallbackReaderWriterImpl read_tag_.Set(call_.call(), [this](bool ok) { reactor_->OnReadDone(ok); - MaybeFinish(); + MaybeFinish(/*from_reaction=*/true); }, &read_ops_, /*can_inline=*/false); read_ops_.set_core_cq_tag(&read_tag_); // Also set up the Finish tag and op set. - finish_tag_.Set(call_.call(), [this](bool /*ok*/) { MaybeFinish(); }, - &finish_ops_, - /*can_inline=*/false); + finish_tag_.Set( + call_.call(), + [this](bool /*ok*/) { MaybeFinish(/*from_reaction=*/true); }, + &finish_ops_, + /*can_inline=*/false); finish_ops_.ClientRecvStatus(context_, &finish_status_); finish_ops_.set_core_cq_tag(&finish_tag_); } @@ -682,7 +718,8 @@ class ClientCallbackReaderImpl : public ClientCallbackReader { // there are no tests catching the compiler warning. static void operator delete(void*, void*) { GPR_CODEGEN_ASSERT(false); } - void MaybeFinish() { + // MaybeFinish behaves as in ClientCallbackReaderWriterImpl. + void MaybeFinish(bool from_reaction) { if (GPR_UNLIKELY(callbacks_outstanding_.fetch_sub( 1, std::memory_order_acq_rel) == 1)) { ::grpc::Status s = std::move(finish_status_); @@ -690,7 +727,11 @@ class ClientCallbackReaderImpl : public ClientCallbackReader { auto* call = call_.call(); this->~ClientCallbackReaderImpl(); ::grpc::g_core_codegen_interface->grpc_call_unref(call); - reactor->OnDone(s); + if (GPR_LIKELY(from_reaction)) { + reactor->OnDone(s); + } else { + reactor->InternalScheduleOnDone(std::move(s)); + } } } @@ -703,7 +744,7 @@ class ClientCallbackReaderImpl : public ClientCallbackReader { start_tag_.Set(call_.call(), [this](bool ok) { reactor_->OnReadInitialMetadataDone(ok); - MaybeFinish(); + MaybeFinish(/*from_reaction=*/true); }, &start_ops_, /*can_inline=*/false); start_ops_.SendInitialMetadata(&context_->send_initial_metadata_, @@ -716,7 +757,7 @@ class ClientCallbackReaderImpl : public ClientCallbackReader { read_tag_.Set(call_.call(), [this](bool ok) { reactor_->OnReadDone(ok); - MaybeFinish(); + MaybeFinish(/*from_reaction=*/true); }, &read_ops_, /*can_inline=*/false); read_ops_.set_core_cq_tag(&read_tag_); @@ -729,8 +770,10 @@ class ClientCallbackReaderImpl : public ClientCallbackReader { started_.store(true, std::memory_order_release); } - finish_tag_.Set(call_.call(), [this](bool /*ok*/) { MaybeFinish(); }, - &finish_ops_, /*can_inline=*/false); + finish_tag_.Set( + call_.call(), + [this](bool /*ok*/) { MaybeFinish(/*from_reaction=*/true); }, + &finish_ops_, /*can_inline=*/false); finish_ops_.ClientRecvStatus(context_, &finish_status_); finish_ops_.set_core_cq_tag(&finish_tag_); call_.PerformOps(&finish_ops_); @@ -752,7 +795,7 @@ class ClientCallbackReaderImpl : public ClientCallbackReader { void AddHold(int holds) override { callbacks_outstanding_.fetch_add(holds, std::memory_order_relaxed); } - void RemoveHold() override { MaybeFinish(); } + void RemoveHold() override { MaybeFinish(/*from_reaction=*/false); } private: friend class ClientCallbackReaderFactory; @@ -833,7 +876,8 @@ class ClientCallbackWriterImpl : public ClientCallbackWriter { // there are no tests catching the compiler warning. static void operator delete(void*, void*) { GPR_CODEGEN_ASSERT(false); } - void MaybeFinish() { + // MaybeFinish behaves as in ClientCallbackReaderWriterImpl. + void MaybeFinish(bool from_reaction) { if (GPR_UNLIKELY(callbacks_outstanding_.fetch_sub( 1, std::memory_order_acq_rel) == 1)) { ::grpc::Status s = std::move(finish_status_); @@ -841,7 +885,11 @@ class ClientCallbackWriterImpl : public ClientCallbackWriter { auto* call = call_.call(); this->~ClientCallbackWriterImpl(); ::grpc::g_core_codegen_interface->grpc_call_unref(call); - reactor->OnDone(s); + if (GPR_LIKELY(from_reaction)) { + reactor->OnDone(s); + } else { + reactor->InternalScheduleOnDone(std::move(s)); + } } } @@ -874,7 +922,7 @@ class ClientCallbackWriterImpl : public ClientCallbackWriter { // MaybeFinish outside the lock to make sure that destruction of this object // doesn't take place while holding the lock (which would cause the lock to // be released after destruction) - this->MaybeFinish(); + this->MaybeFinish(/*from_reaction=*/false); } void Write(const Request* msg, ::grpc::WriteOptions options) override { @@ -907,7 +955,7 @@ class ClientCallbackWriterImpl : public ClientCallbackWriter { writes_done_tag_.Set(call_.call(), [this](bool ok) { reactor_->OnWritesDoneDone(ok); - MaybeFinish(); + MaybeFinish(/*from_reaction=*/true); }, &writes_done_ops_, /*can_inline=*/false); writes_done_ops_.set_core_cq_tag(&writes_done_tag_); @@ -932,7 +980,7 @@ class ClientCallbackWriterImpl : public ClientCallbackWriter { void AddHold(int holds) override { callbacks_outstanding_.fetch_add(holds, std::memory_order_relaxed); } - void RemoveHold() override { MaybeFinish(); } + void RemoveHold() override { MaybeFinish(/*from_reaction=*/false); } private: friend class ClientCallbackWriterFactory; @@ -953,7 +1001,7 @@ class ClientCallbackWriterImpl : public ClientCallbackWriter { start_tag_.Set(call_.call(), [this](bool ok) { reactor_->OnReadInitialMetadataDone(ok); - MaybeFinish(); + MaybeFinish(/*from_reaction=*/true); }, &start_ops_, /*can_inline=*/false); start_ops_.RecvInitialMetadata(context_); @@ -962,7 +1010,7 @@ class ClientCallbackWriterImpl : public ClientCallbackWriter { write_tag_.Set(call_.call(), [this](bool ok) { reactor_->OnWriteDone(ok); - MaybeFinish(); + MaybeFinish(/*from_reaction=*/true); }, &write_ops_, /*can_inline=*/false); write_ops_.set_core_cq_tag(&write_tag_); @@ -970,9 +1018,11 @@ class ClientCallbackWriterImpl : public ClientCallbackWriter { // Also set up the Finish tag and op set. finish_ops_.RecvMessage(response); finish_ops_.AllowNoMessage(); - finish_tag_.Set(call_.call(), [this](bool /*ok*/) { MaybeFinish(); }, - &finish_ops_, - /*can_inline=*/false); + finish_tag_.Set( + call_.call(), + [this](bool /*ok*/) { MaybeFinish(/*from_reaction=*/true); }, + &finish_ops_, + /*can_inline=*/false); finish_ops_.ClientRecvStatus(context_, &finish_status_); finish_ops_.set_core_cq_tag(&finish_tag_); } @@ -1068,12 +1118,16 @@ class ClientCallbackUnaryImpl final : public ClientCallbackUnary { call_.PerformOps(&start_ops_); finish_tag_.Set(call_.call(), [this](bool /*ok*/) { MaybeFinish(); }, - &finish_ops_, /*can_inline=*/false); + &finish_ops_, + /*can_inline=*/false); finish_ops_.ClientRecvStatus(context_, &finish_status_); finish_ops_.set_core_cq_tag(&finish_tag_); call_.PerformOps(&finish_ops_); } + // In the unary case, MaybeFinish is only ever invoked from a + // library-initiated reaction, so it will just directly call OnDone if this is + // the last reaction for this RPC. void MaybeFinish() { if (GPR_UNLIKELY(callbacks_outstanding_.fetch_sub( 1, std::memory_order_acq_rel) == 1)) { diff --git a/src/cpp/client/client_callback.cc b/src/cpp/client/client_callback.cc new file mode 100644 index 00000000000..ca325ba1dbf --- /dev/null +++ b/src/cpp/client/client_callback.cc @@ -0,0 +1,52 @@ +/* + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "src/core/lib/iomgr/closure.h" +#include "src/core/lib/iomgr/exec_ctx.h" +#include "src/core/lib/iomgr/executor.h" + +namespace grpc_impl { +namespace internal { + +void ClientReactor::InternalScheduleOnDone(grpc::Status s) { + // Unlike other uses of closure, do not Ref or Unref here since the reactor + // object's lifetime is controlled by user code. + grpc_core::ExecCtx exec_ctx; + struct ClosureWithArg { + grpc_closure closure; + ClientReactor* const reactor; + const grpc::Status status; + ClosureWithArg(ClientReactor* reactor_arg, grpc::Status s) + : reactor(reactor_arg), status(std::move(s)) { + GRPC_CLOSURE_INIT(&closure, + [](void* void_arg, grpc_error*) { + ClosureWithArg* arg = + static_cast(void_arg); + arg->reactor->OnDone(arg->status); + delete arg; + }, + this, grpc_schedule_on_exec_ctx); + } + }; + ClosureWithArg* arg = new ClosureWithArg(this, std::move(s)); + grpc_core::Executor::Run(&arg->closure, GRPC_ERROR_NONE); +} + +} // namespace internal +} // namespace grpc_impl diff --git a/test/cpp/end2end/client_callback_end2end_test.cc b/test/cpp/end2end/client_callback_end2end_test.cc index cd3c14881bd..f9c2dbaca30 100644 --- a/test/cpp/end2end/client_callback_end2end_test.cc +++ b/test/cpp/end2end/client_callback_end2end_test.cc @@ -1478,6 +1478,12 @@ TEST_P(ClientCallbackEnd2endTest, done_cv_.wait(l); } } + // RemoveHold under the same lock used for OnDone to make sure that we don't + // call OnDone directly or indirectly from the RemoveHold function. + void RemoveHoldUnderLock() { + std::unique_lock l(mu_); + RemoveHold(); + } const Status& status() { std::unique_lock l(mu_); return status_; @@ -1522,7 +1528,7 @@ TEST_P(ClientCallbackEnd2endTest, ++reads_complete; } } - client.RemoveHold(); + client.RemoveHoldUnderLock(); client.Await(); EXPECT_EQ(kServerDefaultResponseStreamsToSend, reads_complete); diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 1867476c484..e69d345dfbb 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -1920,6 +1920,7 @@ src/core/tsi/transport_security_grpc.h \ src/core/tsi/transport_security_interface.h \ src/cpp/README.md \ src/cpp/client/channel_cc.cc \ +src/cpp/client/client_callback.cc \ src/cpp/client/client_context.cc \ src/cpp/client/client_interceptor.cc \ src/cpp/client/create_channel.cc \ From b42402839eb679c8e442a5b4c323f3d393147032 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Tue, 16 Jun 2020 14:01:22 -0700 Subject: [PATCH 65/80] Pass repo manager duty to nicolasnoble --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- .github/ISSUE_TEMPLATE/cleanup_request.md | 2 +- .github/ISSUE_TEMPLATE/feature_request.md | 2 +- .github/ISSUE_TEMPLATE/question.md | 2 +- .github/pull_request_template.md | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 663fcbde3fb..d103ec92af2 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -2,7 +2,7 @@ name: Report a bug about: Create a report to help us improve labels: kind/bug, priority/P2 -assignees: markdroth +assignees: nicolasnoble --- diff --git a/.github/ISSUE_TEMPLATE/cleanup_request.md b/.github/ISSUE_TEMPLATE/cleanup_request.md index cd547e286c1..284797c9e59 100644 --- a/.github/ISSUE_TEMPLATE/cleanup_request.md +++ b/.github/ISSUE_TEMPLATE/cleanup_request.md @@ -2,7 +2,7 @@ name: Request a cleanup about: Suggest a cleanup in our repository labels: kind/internal cleanup, priority/P2 -assignees: markdroth +assignees: nicolasnoble --- diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 717752f9207..f538eae6194 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -2,7 +2,7 @@ name: Request a feature about: Suggest an idea for this project labels: kind/enhancement, priority/P2 -assignees: markdroth +assignees: nicolasnoble --- diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md index 48f55c83d73..da5741aa267 100644 --- a/.github/ISSUE_TEMPLATE/question.md +++ b/.github/ISSUE_TEMPLATE/question.md @@ -2,7 +2,7 @@ name: Ask a question about: Ask a question labels: kind/question, priority/P3 -assignees: markdroth +assignees: nicolasnoble --- diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index a85cfad9c7a..57af6c21597 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -8,4 +8,4 @@ If you know who should review your pull request, please remove the mentioning be --> -@markdroth +@nicolasnoble From 5e77db994b143a282c90b82713dea35cdb15df68 Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Fri, 12 Jun 2020 00:22:15 -0700 Subject: [PATCH 66/80] Add gentle failover test and account for unexpected primary/secondary assignment --- src/proto/grpc/testing/test.proto | 6 + tools/run_tests/run_xds_tests.py | 212 ++++++++++++++++++++++-------- 2 files changed, 166 insertions(+), 52 deletions(-) diff --git a/src/proto/grpc/testing/test.proto b/src/proto/grpc/testing/test.proto index 0b198d8c260..1da43f48989 100644 --- a/src/proto/grpc/testing/test.proto +++ b/src/proto/grpc/testing/test.proto @@ -84,3 +84,9 @@ service LoadBalancerStatsService { rpc GetClientStats(LoadBalancerStatsRequest) returns (LoadBalancerStatsResponse) {} } + +// A service to remotely control health status of an xDS test server. +service XdsUpdateHealthService { + rpc SetServing(grpc.testing.Empty) returns (grpc.testing.Empty); + rpc SetNotServing(grpc.testing.Empty) returns (grpc.testing.Empty); +} diff --git a/tools/run_tests/run_xds_tests.py b/tools/run_tests/run_xds_tests.py index ffb029f5f04..75e6640b336 100755 --- a/tools/run_tests/run_xds_tests.py +++ b/tools/run_tests/run_xds_tests.py @@ -32,6 +32,7 @@ from oauth2client.client import GoogleCredentials import python_utils.jobset as jobset import python_utils.report_utils as report_utils +from src.proto.grpc.testing import empty_pb2 from src.proto.grpc.testing import messages_pb2 from src.proto.grpc.testing import test_pb2_grpc @@ -46,6 +47,7 @@ logger.setLevel(logging.WARNING) _TEST_CASES = [ 'backends_restart', 'change_backend_service', + 'gentle_failover', 'new_instance_group_receives_traffic', 'ping_pong', 'remove_instance_group', @@ -231,12 +233,6 @@ _BOOTSTRAP_TEMPLATE = """ _TESTS_TO_FAIL_ON_RPC_FAILURE = [ 'new_instance_group_receives_traffic', 'ping_pong', 'round_robin' ] -_TESTS_USING_SECONDARY_IG = [ - 'secondary_locality_gets_no_requests_on_partial_primary_failure', - 'secondary_locality_gets_requests_on_primary_failure' -] -_USE_SECONDARY_IG = any( - [t in args.test_case for t in _TESTS_USING_SECONDARY_IG]) _PATH_MATCHER_NAME = 'path-matcher' _BASE_TEMPLATE_NAME = 'test-template' _BASE_INSTANCE_GROUP_NAME = 'test-ig' @@ -267,6 +263,10 @@ def get_client_stats(num_rpcs, timeout_sec): return response +class RpcDistributionError(Exception): + pass + + def _verify_rpcs_to_given_backends(backends, timeout_sec, num_rpcs, allow_failures): start_time = time.time() @@ -287,7 +287,7 @@ def _verify_rpcs_to_given_backends(backends, timeout_sec, num_rpcs, error_msg = '%d RPCs failed' % stats.num_failures if not error_msg: return - raise Exception(error_msg) + raise RpcDistributionError(error_msg) def wait_until_all_rpcs_go_to_given_backends_or_fail(backends, @@ -396,6 +396,59 @@ def test_change_backend_service(gcp, original_backend_service, instance_group, patch_backend_instances(gcp, alternate_backend_service, []) +def test_gentle_failover(gcp, + backend_service, + primary_instance_group, + secondary_instance_group, + swapped_primary_and_secondary=False): + logger.info('Running test_gentle_failover') + num_primary_instances = len(get_instance_names(gcp, primary_instance_group)) + min_instances_for_gentle_failover = 3 # Need >50% failure to start failover + try: + if num_primary_instances < min_instances_for_gentle_failover: + resize_instance_group(gcp, primary_instance_group, + min_instances_for_gentle_failover) + patch_backend_instances( + gcp, backend_service, + [primary_instance_group, secondary_instance_group]) + primary_instance_names = get_instance_names(gcp, primary_instance_group) + secondary_instance_names = get_instance_names(gcp, + secondary_instance_group) + wait_for_healthy_backends(gcp, backend_service, primary_instance_group) + wait_for_healthy_backends(gcp, backend_service, + secondary_instance_group) + wait_until_all_rpcs_go_to_given_backends(primary_instance_names, + _WAIT_FOR_STATS_SEC) + instances_to_stop = primary_instance_names[:-1] + remaining_instances = primary_instance_names[-1:] + try: + set_serving_status(instances_to_stop, + gcp.service_port, + serving=False) + wait_until_all_rpcs_go_to_given_backends( + remaining_instances + secondary_instance_names, + _WAIT_FOR_BACKEND_SEC) + finally: + set_serving_status(primary_instance_names, + gcp.service_port, + serving=True) + except RpcDistributionError as e: + if not swapped_primary_and_secondary and is_primary_instance_group( + gcp, secondary_instance_group): + # Swap expectation of primary and secondary instance groups. + test_gentle_failover(gcp, + backend_service, + secondary_instance_group, + primary_instance_group, + swapped_primary_and_secondary=True) + else: + raise e + finally: + patch_backend_instances(gcp, backend_service, [primary_instance_group]) + resize_instance_group(gcp, primary_instance_group, + num_primary_instances) + + def test_new_instance_group_receives_traffic(gcp, backend_service, instance_group, same_zone_instance_group): @@ -478,61 +531,93 @@ def test_round_robin(gcp, backend_service, instance_group): def test_secondary_locality_gets_no_requests_on_partial_primary_failure( - gcp, backend_service, primary_instance_group, - secondary_zone_instance_group): + gcp, + backend_service, + primary_instance_group, + secondary_instance_group, + swapped_primary_and_secondary=False): logger.info( - 'Running test_secondary_locality_gets_no_requests_on_partial_primary_failure' + 'Running secondary_locality_gets_no_requests_on_partial_primary_failure' ) try: patch_backend_instances( gcp, backend_service, - [primary_instance_group, secondary_zone_instance_group]) + [primary_instance_group, secondary_instance_group]) wait_for_healthy_backends(gcp, backend_service, primary_instance_group) wait_for_healthy_backends(gcp, backend_service, - secondary_zone_instance_group) - primary_instance_names = get_instance_names(gcp, instance_group) - secondary_instance_names = get_instance_names( - gcp, secondary_zone_instance_group) + secondary_instance_group) + primary_instance_names = get_instance_names(gcp, primary_instance_group) wait_until_all_rpcs_go_to_given_backends(primary_instance_names, _WAIT_FOR_STATS_SEC) - original_size = len(primary_instance_names) - resize_instance_group(gcp, primary_instance_group, original_size - 1) - remaining_instance_names = get_instance_names(gcp, - primary_instance_group) - wait_until_all_rpcs_go_to_given_backends(remaining_instance_names, - _WAIT_FOR_BACKEND_SEC) + instances_to_stop = primary_instance_names[:1] + remaining_instances = primary_instance_names[1:] + try: + set_serving_status(instances_to_stop, + gcp.service_port, + serving=False) + wait_until_all_rpcs_go_to_given_backends(remaining_instances, + _WAIT_FOR_BACKEND_SEC) + finally: + set_serving_status(primary_instance_names, + gcp.service_port, + serving=True) + except RpcDistributionError as e: + if not swapped_primary_and_secondary and is_primary_instance_group( + gcp, secondary_instance_group): + # Swap expectation of primary and secondary instance groups. + test_secondary_locality_gets_no_requests_on_partial_primary_failure( + gcp, + backend_service, + secondary_instance_group, + primary_instance_group, + swapped_primary_and_secondary=True) + else: + raise e finally: patch_backend_instances(gcp, backend_service, [primary_instance_group]) - resize_instance_group(gcp, primary_instance_group, original_size) def test_secondary_locality_gets_requests_on_primary_failure( - gcp, backend_service, primary_instance_group, - secondary_zone_instance_group): - logger.info( - 'Running test_secondary_locality_gets_requests_on_primary_failure') + gcp, + backend_service, + primary_instance_group, + secondary_instance_group, + swapped_primary_and_secondary=False): + logger.info('Running secondary_locality_gets_requests_on_primary_failure') try: patch_backend_instances( gcp, backend_service, - [primary_instance_group, secondary_zone_instance_group]) + [primary_instance_group, secondary_instance_group]) wait_for_healthy_backends(gcp, backend_service, primary_instance_group) wait_for_healthy_backends(gcp, backend_service, - secondary_zone_instance_group) - primary_instance_names = get_instance_names(gcp, instance_group) - secondary_instance_names = get_instance_names( - gcp, secondary_zone_instance_group) + secondary_instance_group) + primary_instance_names = get_instance_names(gcp, primary_instance_group) + secondary_instance_names = get_instance_names(gcp, + secondary_instance_group) wait_until_all_rpcs_go_to_given_backends(primary_instance_names, - _WAIT_FOR_BACKEND_SEC) - original_size = len(primary_instance_names) - resize_instance_group(gcp, primary_instance_group, 0) - wait_until_all_rpcs_go_to_given_backends(secondary_instance_names, - _WAIT_FOR_BACKEND_SEC) - - resize_instance_group(gcp, primary_instance_group, original_size) - new_instance_names = get_instance_names(gcp, primary_instance_group) - wait_for_healthy_backends(gcp, backend_service, primary_instance_group) - wait_until_all_rpcs_go_to_given_backends(new_instance_names, - _WAIT_FOR_BACKEND_SEC) + _WAIT_FOR_STATS_SEC) + try: + set_serving_status(primary_instance_names, + gcp.service_port, + serving=False) + wait_until_all_rpcs_go_to_given_backends(secondary_instance_names, + _WAIT_FOR_BACKEND_SEC) + finally: + set_serving_status(primary_instance_names, + gcp.service_port, + serving=True) + except RpcDistributionError as e: + if not swapped_primary_and_secondary and is_primary_instance_group( + gcp, secondary_instance_group): + # Swap expectation of primary and secondary instance groups. + test_secondary_locality_gets_requests_on_primary_failure( + gcp, + backend_service, + secondary_instance_group, + primary_instance_group, + swapped_primary_and_secondary=True) + else: + raise e finally: patch_backend_instances(gcp, backend_service, [primary_instance_group]) @@ -636,6 +721,26 @@ def test_traffic_splitting(gcp, original_backend_service, instance_group, set_validate_for_proxyless(gcp, True) +def set_serving_status(instances, service_port, serving): + for instance in instances: + with grpc.insecure_channel('%s:%d' % + (instance, service_port)) as channel: + stub = test_pb2_grpc.XdsUpdateHealthServiceStub(channel) + if serving: + stub.SetServing(empty_pb2.Empty()) + else: + stub.SetNotServing(empty_pb2.Empty()) + + +def is_primary_instance_group(gcp, instance_group): + # Clients may connect to a TD instance in a different region than the + # client, in which case primary/secondary assignments may not be based on + # the client's actual locality. + instance_names = get_instance_names(gcp, instance_group) + stats = get_client_stats(_NUM_TEST_RPCS, _WAIT_FOR_STATS_SEC) + return all(peer in instance_names for peer in stats.rpcs_by_peer.keys()) + + def get_startup_script(path_to_server_binary, service_port): if path_to_server_binary: return "nohup %s --port=%d 1>/dev/null &" % (path_to_server_binary, @@ -1182,18 +1287,20 @@ def wait_for_healthy_backends(gcp, timeout_sec=_WAIT_FOR_BACKEND_SEC): start_time = time.time() config = {'group': instance_group.url} + expected_size = len(get_instance_names(gcp, instance_group)) while time.time() - start_time <= timeout_sec: result = gcp.compute.backendServices().getHealth( project=gcp.project, backendService=backend_service.name, body=config).execute(num_retries=_GCP_API_RETRIES) if 'healthStatus' in result: + logger.info('received healthStatus: %s', result['healthStatus']) healthy = True for instance in result['healthStatus']: if instance['healthState'] != 'HEALTHY': healthy = False break - if healthy: + if healthy and expected_size == len(result['healthStatus']): return time.sleep(2) raise Exception('Not all backends became healthy within %d seconds: %s' % @@ -1227,6 +1334,7 @@ def get_instance_names(gcp, instance_group): # just extract the name manually. instance_name = item['instance'].split('/')[-1] instance_names.append(instance_name) + logger.info('retrieved instance names: %s', instance_names) return instance_names @@ -1306,8 +1414,7 @@ try: template_name = _BASE_TEMPLATE_NAME + args.gcp_suffix instance_group_name = _BASE_INSTANCE_GROUP_NAME + args.gcp_suffix same_zone_instance_group_name = _BASE_INSTANCE_GROUP_NAME + '-same-zone' + args.gcp_suffix - if _USE_SECONDARY_IG: - secondary_zone_instance_group_name = _BASE_INSTANCE_GROUP_NAME + '-secondary-zone' + args.gcp_suffix + secondary_zone_instance_group_name = _BASE_INSTANCE_GROUP_NAME + '-secondary-zone' + args.gcp_suffix if args.use_existing_gcp_resources: logger.info('Reusing existing GCP resources') get_health_check(gcp, health_check_name) @@ -1328,9 +1435,8 @@ try: instance_group = get_instance_group(gcp, args.zone, instance_group_name) same_zone_instance_group = get_instance_group( gcp, args.zone, same_zone_instance_group_name) - if _USE_SECONDARY_IG: - secondary_zone_instance_group = get_instance_group( - gcp, args.secondary_zone, secondary_zone_instance_group_name) + secondary_zone_instance_group = get_instance_group( + gcp, args.secondary_zone, secondary_zone_instance_group_name) else: create_health_check(gcp, health_check_name) create_health_check_firewall_rule(gcp, firewall_name) @@ -1360,10 +1466,9 @@ try: patch_backend_instances(gcp, backend_service, [instance_group]) same_zone_instance_group = add_instance_group( gcp, args.zone, same_zone_instance_group_name, _INSTANCE_GROUP_SIZE) - if _USE_SECONDARY_IG: - secondary_zone_instance_group = add_instance_group( - gcp, args.secondary_zone, secondary_zone_instance_group_name, - _INSTANCE_GROUP_SIZE) + secondary_zone_instance_group = add_instance_group( + gcp, args.secondary_zone, secondary_zone_instance_group_name, + _INSTANCE_GROUP_SIZE) wait_for_healthy_backends(gcp, backend_service, instance_group) @@ -1420,6 +1525,9 @@ try: instance_group, alternate_backend_service, same_zone_instance_group) + elif test_case == 'gentle_failover': + test_gentle_failover(gcp, backend_service, instance_group, + secondary_zone_instance_group) elif test_case == 'new_instance_group_receives_traffic': test_new_instance_group_receives_traffic( gcp, backend_service, instance_group, From 3c90fe7bdf91467655066ec65717ce48dbb5ea95 Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Tue, 16 Jun 2020 17:38:24 -0700 Subject: [PATCH 67/80] wait for servers to become healthy after failover test --- tools/run_tests/run_xds_tests.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/run_tests/run_xds_tests.py b/tools/run_tests/run_xds_tests.py index 75e6640b336..81c913f5795 100755 --- a/tools/run_tests/run_xds_tests.py +++ b/tools/run_tests/run_xds_tests.py @@ -447,6 +447,9 @@ def test_gentle_failover(gcp, patch_backend_instances(gcp, backend_service, [primary_instance_group]) resize_instance_group(gcp, primary_instance_group, num_primary_instances) + instance_names = get_instance_names(gcp, primary_instance_group) + wait_until_all_rpcs_go_to_given_backends(instance_names, + _WAIT_FOR_BACKEND_SEC) def test_new_instance_group_receives_traffic(gcp, backend_service, From 6db7c62a13c0663545d8ca4331b06a80ad54129b Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Thu, 28 May 2020 15:44:39 -0700 Subject: [PATCH 68/80] Roll-forward: Change server_finishes_request test to return OK status --- test/core/end2end/tests/server_finishes_request.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/core/end2end/tests/server_finishes_request.cc b/test/core/end2end/tests/server_finishes_request.cc index 5e7f3d96b4a..dc489a4c7fb 100644 --- a/test/core/end2end/tests/server_finishes_request.cc +++ b/test/core/end2end/tests/server_finishes_request.cc @@ -151,7 +151,7 @@ static void simple_request_body(grpc_end2end_test_config /*config*/, op++; op->op = GRPC_OP_SEND_STATUS_FROM_SERVER; op->data.send_status_from_server.trailing_metadata_count = 0; - op->data.send_status_from_server.status = GRPC_STATUS_UNIMPLEMENTED; + op->data.send_status_from_server.status = GRPC_STATUS_OK; grpc_slice status_details = grpc_slice_from_static_string("xyz"); op->data.send_status_from_server.status_details = &status_details; op->flags = 0; @@ -170,7 +170,7 @@ static void simple_request_body(grpc_end2end_test_config /*config*/, CQ_EXPECT_COMPLETION(cqv, tag(1), 1); cq_verify(cqv); - GPR_ASSERT(status == GRPC_STATUS_UNIMPLEMENTED); + GPR_ASSERT(status == GRPC_STATUS_OK); GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz")); GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/foo")); GPR_ASSERT(was_cancelled == 0); From 5ef722a2c9b75186bc51c6401f4f620f926a78f4 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Wed, 17 Jun 2020 12:06:55 -0700 Subject: [PATCH 69/80] Privatize MaybeFinish --- .../impl/codegen/client_callback_impl.h | 142 +++++++++--------- 1 file changed, 71 insertions(+), 71 deletions(-) diff --git a/include/grpcpp/impl/codegen/client_callback_impl.h b/include/grpcpp/impl/codegen/client_callback_impl.h index 3ac46b788ea..bed52fe6a80 100644 --- a/include/grpcpp/impl/codegen/client_callback_impl.h +++ b/include/grpcpp/impl/codegen/client_callback_impl.h @@ -468,28 +468,6 @@ class ClientCallbackReaderWriterImpl // there are no tests catching the compiler warning. static void operator delete(void*, void*) { GPR_CODEGEN_ASSERT(false); } - // MaybeFinish can be called from reactions or from user-initiated operations - // like StartCall or RemoveHold. If this is the last operation or hold on this - // object, it will invoke the OnDone reaction. If MaybeFinish was called from - // a reaction, it can call OnDone directly. If not, it would need to schedule - // OnDone onto an executor thread to avoid the possibility of deadlocking with - // any locks in the user code that invoked it. - void MaybeFinish(bool from_reaction) { - if (GPR_UNLIKELY(callbacks_outstanding_.fetch_sub( - 1, std::memory_order_acq_rel) == 1)) { - ::grpc::Status s = std::move(finish_status_); - auto* reactor = reactor_; - auto* call = call_.call(); - this->~ClientCallbackReaderWriterImpl(); - ::grpc::g_core_codegen_interface->grpc_call_unref(call); - if (GPR_LIKELY(from_reaction)) { - reactor->OnDone(s); - } else { - reactor->InternalScheduleOnDone(std::move(s)); - } - } - } - void StartCall() override { // This call initiates two batches, plus any backlog, each with a callback // 1. Send initial metadata (unless corked) + recv initial metadata @@ -641,6 +619,28 @@ class ClientCallbackReaderWriterImpl finish_ops_.set_core_cq_tag(&finish_tag_); } + // MaybeFinish can be called from reactions or from user-initiated operations + // like StartCall or RemoveHold. If this is the last operation or hold on this + // object, it will invoke the OnDone reaction. If MaybeFinish was called from + // a reaction, it can call OnDone directly. If not, it would need to schedule + // OnDone onto an executor thread to avoid the possibility of deadlocking with + // any locks in the user code that invoked it. + void MaybeFinish(bool from_reaction) { + if (GPR_UNLIKELY(callbacks_outstanding_.fetch_sub( + 1, std::memory_order_acq_rel) == 1)) { + ::grpc::Status s = std::move(finish_status_); + auto* reactor = reactor_; + auto* call = call_.call(); + this->~ClientCallbackReaderWriterImpl(); + ::grpc::g_core_codegen_interface->grpc_call_unref(call); + if (GPR_LIKELY(from_reaction)) { + reactor->OnDone(s); + } else { + reactor->InternalScheduleOnDone(std::move(s)); + } + } + } + ::grpc_impl::ClientContext* const context_; grpc::internal::Call call_; ClientBidiReactor* const reactor_; @@ -718,23 +718,6 @@ class ClientCallbackReaderImpl : public ClientCallbackReader { // there are no tests catching the compiler warning. static void operator delete(void*, void*) { GPR_CODEGEN_ASSERT(false); } - // MaybeFinish behaves as in ClientCallbackReaderWriterImpl. - void MaybeFinish(bool from_reaction) { - if (GPR_UNLIKELY(callbacks_outstanding_.fetch_sub( - 1, std::memory_order_acq_rel) == 1)) { - ::grpc::Status s = std::move(finish_status_); - auto* reactor = reactor_; - auto* call = call_.call(); - this->~ClientCallbackReaderImpl(); - ::grpc::g_core_codegen_interface->grpc_call_unref(call); - if (GPR_LIKELY(from_reaction)) { - reactor->OnDone(s); - } else { - reactor->InternalScheduleOnDone(std::move(s)); - } - } - } - void StartCall() override { // This call initiates two batches, plus any backlog, each with a callback // 1. Send initial metadata (unless corked) + recv initial metadata @@ -812,6 +795,23 @@ class ClientCallbackReaderImpl : public ClientCallbackReader { start_ops_.ClientSendClose(); } + // MaybeFinish behaves as in ClientCallbackReaderWriterImpl. + void MaybeFinish(bool from_reaction) { + if (GPR_UNLIKELY(callbacks_outstanding_.fetch_sub( + 1, std::memory_order_acq_rel) == 1)) { + ::grpc::Status s = std::move(finish_status_); + auto* reactor = reactor_; + auto* call = call_.call(); + this->~ClientCallbackReaderImpl(); + ::grpc::g_core_codegen_interface->grpc_call_unref(call); + if (GPR_LIKELY(from_reaction)) { + reactor->OnDone(s); + } else { + reactor->InternalScheduleOnDone(std::move(s)); + } + } + } + ::grpc_impl::ClientContext* const context_; grpc::internal::Call call_; ClientReadReactor* const reactor_; @@ -876,23 +876,6 @@ class ClientCallbackWriterImpl : public ClientCallbackWriter { // there are no tests catching the compiler warning. static void operator delete(void*, void*) { GPR_CODEGEN_ASSERT(false); } - // MaybeFinish behaves as in ClientCallbackReaderWriterImpl. - void MaybeFinish(bool from_reaction) { - if (GPR_UNLIKELY(callbacks_outstanding_.fetch_sub( - 1, std::memory_order_acq_rel) == 1)) { - ::grpc::Status s = std::move(finish_status_); - auto* reactor = reactor_; - auto* call = call_.call(); - this->~ClientCallbackWriterImpl(); - ::grpc::g_core_codegen_interface->grpc_call_unref(call); - if (GPR_LIKELY(from_reaction)) { - reactor->OnDone(s); - } else { - reactor->InternalScheduleOnDone(std::move(s)); - } - } - } - void StartCall() override { // This call initiates two batches, plus any backlog, each with a callback // 1. Send initial metadata (unless corked) + recv initial metadata @@ -1027,6 +1010,23 @@ class ClientCallbackWriterImpl : public ClientCallbackWriter { finish_ops_.set_core_cq_tag(&finish_tag_); } + // MaybeFinish behaves as in ClientCallbackReaderWriterImpl. + void MaybeFinish(bool from_reaction) { + if (GPR_UNLIKELY(callbacks_outstanding_.fetch_sub( + 1, std::memory_order_acq_rel) == 1)) { + ::grpc::Status s = std::move(finish_status_); + auto* reactor = reactor_; + auto* call = call_.call(); + this->~ClientCallbackWriterImpl(); + ::grpc::g_core_codegen_interface->grpc_call_unref(call); + if (GPR_LIKELY(from_reaction)) { + reactor->OnDone(s); + } else { + reactor->InternalScheduleOnDone(std::move(s)); + } + } + } + ::grpc_impl::ClientContext* const context_; grpc::internal::Call call_; ClientWriteReactor* const reactor_; @@ -1125,21 +1125,6 @@ class ClientCallbackUnaryImpl final : public ClientCallbackUnary { call_.PerformOps(&finish_ops_); } - // In the unary case, MaybeFinish is only ever invoked from a - // library-initiated reaction, so it will just directly call OnDone if this is - // the last reaction for this RPC. - void MaybeFinish() { - if (GPR_UNLIKELY(callbacks_outstanding_.fetch_sub( - 1, std::memory_order_acq_rel) == 1)) { - ::grpc::Status s = std::move(finish_status_); - auto* reactor = reactor_; - auto* call = call_.call(); - this->~ClientCallbackUnaryImpl(); - ::grpc::g_core_codegen_interface->grpc_call_unref(call); - reactor->OnDone(s); - } - } - private: friend class ClientCallbackUnaryFactory; @@ -1156,6 +1141,21 @@ class ClientCallbackUnaryImpl final : public ClientCallbackUnary { finish_ops_.AllowNoMessage(); } + // In the unary case, MaybeFinish is only ever invoked from a + // library-initiated reaction, so it will just directly call OnDone if this is + // the last reaction for this RPC. + void MaybeFinish() { + if (GPR_UNLIKELY(callbacks_outstanding_.fetch_sub( + 1, std::memory_order_acq_rel) == 1)) { + ::grpc::Status s = std::move(finish_status_); + auto* reactor = reactor_; + auto* call = call_.call(); + this->~ClientCallbackUnaryImpl(); + ::grpc::g_core_codegen_interface->grpc_call_unref(call); + reactor->OnDone(s); + } + } + ::grpc_impl::ClientContext* const context_; grpc::internal::Call call_; ClientUnaryReactor* const reactor_; From e4bdd326e287793b76fd314299fd9421ecfe685a Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Wed, 17 Jun 2020 12:13:15 -0700 Subject: [PATCH 70/80] Fix buildfier --- src/objective-c/BUILD | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/objective-c/BUILD b/src/objective-c/BUILD index 122c0fa3315..27930862e80 100644 --- a/src/objective-c/BUILD +++ b/src/objective-c/BUILD @@ -14,12 +14,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("//bazel:grpc_build_system.bzl", "grpc_generate_objc_one_off_targets", "grpc_objc_library") +load("@build_bazel_rules_apple//apple:resources.bzl", "apple_resource_bundle") + licenses(["notice"]) # Apache v2 package(default_visibility = ["//visibility:public"]) -load("//bazel:grpc_build_system.bzl", "grpc_generate_objc_one_off_targets", "grpc_objc_library") - exports_files(["LICENSE"]) grpc_generate_objc_one_off_targets() @@ -195,8 +196,6 @@ grpc_objc_library( ], ) -load("@build_bazel_rules_apple//apple:resources.bzl", "apple_resource_bundle") - apple_resource_bundle( # The choice of name is signicant here, since it determines the bundle name. name = "gRPCCertificates", From d74e52ad8509309ee6c338d7147ad2740a673fa7 Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Wed, 17 Jun 2020 22:52:03 -0700 Subject: [PATCH 71/80] Use grpc health checks for PHP and Ruby xDS tests The test suite now dynamically updates the server's health status (via the gRPC health service) to simulate failing backends; this requires grpc health checks, which means the --use_stable_apis flag is no longer compatible with these checks (it should probably be removed, but will leave that for a follow-up). --- tools/internal_ci/linux/grpc_xds_php_test_in_docker.sh | 1 - tools/internal_ci/linux/grpc_xds_ruby_test_in_docker.sh | 1 - tools/run_tests/run_xds_tests.py | 6 +++++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/internal_ci/linux/grpc_xds_php_test_in_docker.sh b/tools/internal_ci/linux/grpc_xds_php_test_in_docker.sh index b6b1b168a98..0d1deffcf5e 100755 --- a/tools/internal_ci/linux/grpc_xds_php_test_in_docker.sh +++ b/tools/internal_ci/linux/grpc_xds_php_test_in_docker.sh @@ -65,6 +65,5 @@ GRPC_VERBOSITY=debug GRPC_TRACE=xds_client,xds_resolver,xds_routing_lb,cds_lb,ed --source_image=projects/grpc-testing/global/images/xds-test-server \ --path_to_server_binary=/java_server/grpc-java/interop-testing/build/install/grpc-interop-testing/bin/xds-test-server \ --gcp_suffix=$(date '+%s') \ - --only_stable_gcp_apis \ --verbose \ --client_cmd='php -d extension=grpc.so -d extension=pthreads.so src/php/tests/interop/xds_client.php --server=xds:///{server_uri} --stats_port={stats_port} --qps={qps}' diff --git a/tools/internal_ci/linux/grpc_xds_ruby_test_in_docker.sh b/tools/internal_ci/linux/grpc_xds_ruby_test_in_docker.sh index 6f381729fa9..c88f6e3e5f3 100644 --- a/tools/internal_ci/linux/grpc_xds_ruby_test_in_docker.sh +++ b/tools/internal_ci/linux/grpc_xds_ruby_test_in_docker.sh @@ -55,6 +55,5 @@ GRPC_VERBOSITY=debug GRPC_TRACE=xds_client,xds_resolver,xds_routing_lb,cds_lb,ed --source_image=projects/grpc-testing/global/images/xds-test-server \ --path_to_server_binary=/java_server/grpc-java/interop-testing/build/install/grpc-interop-testing/bin/xds-test-server \ --gcp_suffix=$(date '+%s') \ - --only_stable_gcp_apis \ --verbose \ --client_cmd='ruby src/ruby/pb/test/xds_client.rb --server=xds:///{server_uri} --stats_port={stats_port} --qps={qps}' diff --git a/tools/run_tests/run_xds_tests.py b/tools/run_tests/run_xds_tests.py index 81c913f5795..3073d301ded 100755 --- a/tools/run_tests/run_xds_tests.py +++ b/tools/run_tests/run_xds_tests.py @@ -186,8 +186,12 @@ argp.add_argument('--log_client_output', help='Log captured client output', default=False, action='store_true') +# TODO(ericgribkoff) Remove this flag once all test environments are verified to +# have access to the alpha compute APIs. argp.add_argument('--only_stable_gcp_apis', - help='Do not use alpha compute APIs', + help='Do not use alpha compute APIs. Some tests may be ' + 'incompatible with this option (gRPC health checks are ' + 'currently alpha and required for simulating server failure', default=False, action='store_true') args = argp.parse_args() From e994c9550265204ed662c0ef9b25b83374898937 Mon Sep 17 00:00:00 2001 From: yulin-liang Date: Thu, 18 Jun 2020 11:50:04 -0700 Subject: [PATCH 72/80] Fix flaky test:ios-test-cronettests --- .../xcshareddata/xcschemes/CronetTests.xcscheme | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/objective-c/tests/Tests.xcodeproj/xcshareddata/xcschemes/CronetTests.xcscheme b/src/objective-c/tests/Tests.xcodeproj/xcshareddata/xcschemes/CronetTests.xcscheme index 0156c906971..16d3d4e62fd 100644 --- a/src/objective-c/tests/Tests.xcodeproj/xcshareddata/xcschemes/CronetTests.xcscheme +++ b/src/objective-c/tests/Tests.xcodeproj/xcshareddata/xcschemes/CronetTests.xcscheme @@ -44,8 +44,6 @@ - - - - + + + + Date: Thu, 18 Jun 2020 12:45:02 -0700 Subject: [PATCH 73/80] Add ConfigSelector API. --- BUILD | 2 + BUILD.gn | 2 + CMakeLists.txt | 2 + Makefile | 2 + build_autogenerated.yaml | 4 + config.m4 | 1 + config.w32 | 1 + gRPC-C++.podspec | 2 + gRPC-Core.podspec | 3 + grpc.gemspec | 2 + grpc.gyp | 2 + package.xml | 2 + .../filters/client_channel/client_channel.cc | 588 +++++++++++------- .../filters/client_channel/config_selector.cc | 62 ++ .../filters/client_channel/config_selector.h | 91 +++ .../client_channel/resolving_lb_policy.cc | 61 +- .../client_channel/resolving_lb_policy.h | 57 +- src/python/grpcio/grpc_core_dependencies.py | 1 + tools/doxygen/Doxyfile.c++.internal | 2 + tools/doxygen/Doxyfile.core.internal | 2 + 20 files changed, 603 insertions(+), 286 deletions(-) create mode 100644 src/core/ext/filters/client_channel/config_selector.cc create mode 100644 src/core/ext/filters/client_channel/config_selector.h diff --git a/BUILD b/BUILD index 99750d060cc..48c1c1cffb7 100644 --- a/BUILD +++ b/BUILD @@ -1028,6 +1028,7 @@ grpc_cc_library( "src/core/ext/filters/client_channel/client_channel_channelz.cc", "src/core/ext/filters/client_channel/client_channel_factory.cc", "src/core/ext/filters/client_channel/client_channel_plugin.cc", + "src/core/ext/filters/client_channel/config_selector.cc", "src/core/ext/filters/client_channel/global_subchannel_pool.cc", "src/core/ext/filters/client_channel/health/health_check_client.cc", "src/core/ext/filters/client_channel/http_connect_handshaker.cc", @@ -1055,6 +1056,7 @@ grpc_cc_library( "src/core/ext/filters/client_channel/client_channel.h", "src/core/ext/filters/client_channel/client_channel_channelz.h", "src/core/ext/filters/client_channel/client_channel_factory.h", + "src/core/ext/filters/client_channel/config_selector.h", "src/core/ext/filters/client_channel/connector.h", "src/core/ext/filters/client_channel/global_subchannel_pool.h", "src/core/ext/filters/client_channel/health/health_check_client.h", diff --git a/BUILD.gn b/BUILD.gn index 378250dd2c0..2d9c2593a5a 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -211,6 +211,8 @@ config("grpc_config") { "src/core/ext/filters/client_channel/client_channel_factory.cc", "src/core/ext/filters/client_channel/client_channel_factory.h", "src/core/ext/filters/client_channel/client_channel_plugin.cc", + "src/core/ext/filters/client_channel/config_selector.cc", + "src/core/ext/filters/client_channel/config_selector.h", "src/core/ext/filters/client_channel/connector.h", "src/core/ext/filters/client_channel/global_subchannel_pool.cc", "src/core/ext/filters/client_channel/global_subchannel_pool.h", diff --git a/CMakeLists.txt b/CMakeLists.txt index 4e0c695a5bc..3666c7cdcb1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1328,6 +1328,7 @@ add_library(grpc src/core/ext/filters/client_channel/client_channel_channelz.cc src/core/ext/filters/client_channel/client_channel_factory.cc src/core/ext/filters/client_channel/client_channel_plugin.cc + src/core/ext/filters/client_channel/config_selector.cc src/core/ext/filters/client_channel/global_subchannel_pool.cc src/core/ext/filters/client_channel/health/health_check_client.cc src/core/ext/filters/client_channel/http_connect_handshaker.cc @@ -1999,6 +2000,7 @@ add_library(grpc_unsecure src/core/ext/filters/client_channel/client_channel_channelz.cc src/core/ext/filters/client_channel/client_channel_factory.cc src/core/ext/filters/client_channel/client_channel_plugin.cc + src/core/ext/filters/client_channel/config_selector.cc src/core/ext/filters/client_channel/global_subchannel_pool.cc src/core/ext/filters/client_channel/health/health_check_client.cc src/core/ext/filters/client_channel/http_connect_handshaker.cc diff --git a/Makefile b/Makefile index 83c7600a698..a57b45bfd02 100644 --- a/Makefile +++ b/Makefile @@ -3630,6 +3630,7 @@ LIBGRPC_SRC = \ src/core/ext/filters/client_channel/client_channel_channelz.cc \ src/core/ext/filters/client_channel/client_channel_factory.cc \ src/core/ext/filters/client_channel/client_channel_plugin.cc \ + src/core/ext/filters/client_channel/config_selector.cc \ src/core/ext/filters/client_channel/global_subchannel_pool.cc \ src/core/ext/filters/client_channel/health/health_check_client.cc \ src/core/ext/filters/client_channel/http_connect_handshaker.cc \ @@ -4275,6 +4276,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/ext/filters/client_channel/client_channel_channelz.cc \ src/core/ext/filters/client_channel/client_channel_factory.cc \ src/core/ext/filters/client_channel/client_channel_plugin.cc \ + src/core/ext/filters/client_channel/config_selector.cc \ src/core/ext/filters/client_channel/global_subchannel_pool.cc \ src/core/ext/filters/client_channel/health/health_check_client.cc \ src/core/ext/filters/client_channel/http_connect_handshaker.cc \ diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index 5f09c7b3e73..e71ab653e9e 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -377,6 +377,7 @@ libs: - src/core/ext/filters/client_channel/client_channel.h - src/core/ext/filters/client_channel/client_channel_channelz.h - src/core/ext/filters/client_channel/client_channel_factory.h + - src/core/ext/filters/client_channel/config_selector.h - src/core/ext/filters/client_channel/connector.h - src/core/ext/filters/client_channel/global_subchannel_pool.h - src/core/ext/filters/client_channel/health/health_check_client.h @@ -746,6 +747,7 @@ libs: - src/core/ext/filters/client_channel/client_channel_channelz.cc - src/core/ext/filters/client_channel/client_channel_factory.cc - src/core/ext/filters/client_channel/client_channel_plugin.cc + - src/core/ext/filters/client_channel/config_selector.cc - src/core/ext/filters/client_channel/global_subchannel_pool.cc - src/core/ext/filters/client_channel/health/health_check_client.cc - src/core/ext/filters/client_channel/http_connect_handshaker.cc @@ -1300,6 +1302,7 @@ libs: - src/core/ext/filters/client_channel/client_channel.h - src/core/ext/filters/client_channel/client_channel_channelz.h - src/core/ext/filters/client_channel/client_channel_factory.h + - src/core/ext/filters/client_channel/config_selector.h - src/core/ext/filters/client_channel/connector.h - src/core/ext/filters/client_channel/global_subchannel_pool.h - src/core/ext/filters/client_channel/health/health_check_client.h @@ -1605,6 +1608,7 @@ libs: - src/core/ext/filters/client_channel/client_channel_channelz.cc - src/core/ext/filters/client_channel/client_channel_factory.cc - src/core/ext/filters/client_channel/client_channel_plugin.cc + - src/core/ext/filters/client_channel/config_selector.cc - src/core/ext/filters/client_channel/global_subchannel_pool.cc - src/core/ext/filters/client_channel/health/health_check_client.cc - src/core/ext/filters/client_channel/http_connect_handshaker.cc diff --git a/config.m4 b/config.m4 index cd1019abcd9..5ee9f1c8d58 100644 --- a/config.m4 +++ b/config.m4 @@ -45,6 +45,7 @@ if test "$PHP_GRPC" != "no"; then src/core/ext/filters/client_channel/client_channel_channelz.cc \ src/core/ext/filters/client_channel/client_channel_factory.cc \ src/core/ext/filters/client_channel/client_channel_plugin.cc \ + src/core/ext/filters/client_channel/config_selector.cc \ src/core/ext/filters/client_channel/global_subchannel_pool.cc \ src/core/ext/filters/client_channel/health/health_check_client.cc \ src/core/ext/filters/client_channel/http_connect_handshaker.cc \ diff --git a/config.w32 b/config.w32 index 8109f9103af..1c533a33302 100644 --- a/config.w32 +++ b/config.w32 @@ -14,6 +14,7 @@ if (PHP_GRPC != "no") { "src\\core\\ext\\filters\\client_channel\\client_channel_channelz.cc " + "src\\core\\ext\\filters\\client_channel\\client_channel_factory.cc " + "src\\core\\ext\\filters\\client_channel\\client_channel_plugin.cc " + + "src\\core\\ext\\filters\\client_channel\\config_selector.cc " + "src\\core\\ext\\filters\\client_channel\\global_subchannel_pool.cc " + "src\\core\\ext\\filters\\client_channel\\health\\health_check_client.cc " + "src\\core\\ext\\filters\\client_channel\\http_connect_handshaker.cc " + diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index 8851119c07f..f34c2cb79a7 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -227,6 +227,7 @@ Pod::Spec.new do |s| 'src/core/ext/filters/client_channel/client_channel.h', 'src/core/ext/filters/client_channel/client_channel_channelz.h', 'src/core/ext/filters/client_channel/client_channel_factory.h', + 'src/core/ext/filters/client_channel/config_selector.h', 'src/core/ext/filters/client_channel/connector.h', 'src/core/ext/filters/client_channel/global_subchannel_pool.h', 'src/core/ext/filters/client_channel/health/health_check_client.h', @@ -687,6 +688,7 @@ Pod::Spec.new do |s| 'src/core/ext/filters/client_channel/client_channel.h', 'src/core/ext/filters/client_channel/client_channel_channelz.h', 'src/core/ext/filters/client_channel/client_channel_factory.h', + 'src/core/ext/filters/client_channel/config_selector.h', 'src/core/ext/filters/client_channel/connector.h', 'src/core/ext/filters/client_channel/global_subchannel_pool.h', 'src/core/ext/filters/client_channel/health/health_check_client.h', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 695768422c2..aaf5609a9c3 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -195,6 +195,8 @@ Pod::Spec.new do |s| 'src/core/ext/filters/client_channel/client_channel_factory.cc', 'src/core/ext/filters/client_channel/client_channel_factory.h', 'src/core/ext/filters/client_channel/client_channel_plugin.cc', + 'src/core/ext/filters/client_channel/config_selector.cc', + 'src/core/ext/filters/client_channel/config_selector.h', 'src/core/ext/filters/client_channel/connector.h', 'src/core/ext/filters/client_channel/global_subchannel_pool.cc', 'src/core/ext/filters/client_channel/global_subchannel_pool.h', @@ -1051,6 +1053,7 @@ Pod::Spec.new do |s| 'src/core/ext/filters/client_channel/client_channel.h', 'src/core/ext/filters/client_channel/client_channel_channelz.h', 'src/core/ext/filters/client_channel/client_channel_factory.h', + 'src/core/ext/filters/client_channel/config_selector.h', 'src/core/ext/filters/client_channel/connector.h', 'src/core/ext/filters/client_channel/global_subchannel_pool.h', 'src/core/ext/filters/client_channel/health/health_check_client.h', diff --git a/grpc.gemspec b/grpc.gemspec index af04b9d9de4..881f55a81c2 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -117,6 +117,8 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/filters/client_channel/client_channel_factory.cc ) s.files += %w( src/core/ext/filters/client_channel/client_channel_factory.h ) s.files += %w( src/core/ext/filters/client_channel/client_channel_plugin.cc ) + s.files += %w( src/core/ext/filters/client_channel/config_selector.cc ) + s.files += %w( src/core/ext/filters/client_channel/config_selector.h ) s.files += %w( src/core/ext/filters/client_channel/connector.h ) s.files += %w( src/core/ext/filters/client_channel/global_subchannel_pool.cc ) s.files += %w( src/core/ext/filters/client_channel/global_subchannel_pool.h ) diff --git a/grpc.gyp b/grpc.gyp index 4022227f5e3..e8bed76e891 100644 --- a/grpc.gyp +++ b/grpc.gyp @@ -440,6 +440,7 @@ 'src/core/ext/filters/client_channel/client_channel_channelz.cc', 'src/core/ext/filters/client_channel/client_channel_factory.cc', 'src/core/ext/filters/client_channel/client_channel_plugin.cc', + 'src/core/ext/filters/client_channel/config_selector.cc', 'src/core/ext/filters/client_channel/global_subchannel_pool.cc', 'src/core/ext/filters/client_channel/health/health_check_client.cc', 'src/core/ext/filters/client_channel/http_connect_handshaker.cc', @@ -947,6 +948,7 @@ 'src/core/ext/filters/client_channel/client_channel_channelz.cc', 'src/core/ext/filters/client_channel/client_channel_factory.cc', 'src/core/ext/filters/client_channel/client_channel_plugin.cc', + 'src/core/ext/filters/client_channel/config_selector.cc', 'src/core/ext/filters/client_channel/global_subchannel_pool.cc', 'src/core/ext/filters/client_channel/health/health_check_client.cc', 'src/core/ext/filters/client_channel/http_connect_handshaker.cc', diff --git a/package.xml b/package.xml index 0166a778789..262c66a6ac3 100644 --- a/package.xml +++ b/package.xml @@ -97,6 +97,8 @@ + + diff --git a/src/core/ext/filters/client_channel/client_channel.cc b/src/core/ext/filters/client_channel/client_channel.cc index d8af25d8f2f..8533be0fe2b 100644 --- a/src/core/ext/filters/client_channel/client_channel.cc +++ b/src/core/ext/filters/client_channel/client_channel.cc @@ -40,6 +40,7 @@ #include "src/core/ext/filters/client_channel/backend_metric.h" #include "src/core/ext/filters/client_channel/backup_poller.h" +#include "src/core/ext/filters/client_channel/config_selector.h" #include "src/core/ext/filters/client_channel/global_subchannel_pool.h" #include "src/core/ext/filters/client_channel/http_connect_handshaker.h" #include "src/core/ext/filters/client_channel/lb_policy_registry.h" @@ -149,12 +150,16 @@ class ChannelData { bool received_service_config_data() const { return received_service_config_data_; } + grpc_error* resolver_transient_failure_error() const { + return resolver_transient_failure_error_; + } RefCountedPtr retry_throttle_data() const { return retry_throttle_data_; } RefCountedPtr service_config() const { return service_config_; } + ConfigSelector* config_selector() const { return config_selector_.get(); } WorkSerializer* work_serializer() const { return work_serializer_.get(); } RefCountedPtr GetConnectedSubchannelInDataPlane( @@ -234,6 +239,29 @@ class ChannelData { Atomic done_{false}; }; + class ChannelConfigHelper + : public ResolvingLoadBalancingPolicy::ChannelConfigHelper { + public: + explicit ChannelConfigHelper(ChannelData* chand) : chand_(chand) {} + + ApplyServiceConfigResult ApplyServiceConfig( + const Resolver::Result& result) override; + + void ApplyConfigSelector( + bool service_config_changed, + RefCountedPtr config_selector) override; + + void ResolverTransientFailure(grpc_error* error) override; + + private: + static void ProcessLbPolicy( + const Resolver::Result& resolver_result, + const internal::ClientChannelGlobalParsedConfig* parsed_service_config, + RefCountedPtr* lb_policy_config); + + ChannelData* chand_; + }; + ChannelData(grpc_channel_element_args* args, grpc_error** error); ~ChannelData(); @@ -241,30 +269,20 @@ class ChannelData { grpc_connectivity_state state, const char* reason, std::unique_ptr picker); - void UpdateServiceConfigLocked( - RefCountedPtr retry_throttle_data, - RefCountedPtr service_config); + void UpdateServiceConfigInDataPlaneLocked( + bool service_config_changed, + RefCountedPtr config_selector); void CreateResolvingLoadBalancingPolicyLocked(); void DestroyResolvingLoadBalancingPolicyLocked(); - static bool ProcessResolverResultLocked( - void* arg, const Resolver::Result& result, - RefCountedPtr* lb_policy_config, - grpc_error** service_config_error, bool* no_valid_service_config); - grpc_error* DoPingLocked(grpc_transport_op* op); void StartTransportOpLocked(grpc_transport_op* op); void TryToConnectLocked(); - void ProcessLbPolicy( - const Resolver::Result& resolver_result, - const internal::ClientChannelGlobalParsedConfig* parsed_service_config, - RefCountedPtr* lb_policy_config); - // // Fields set at construction and never modified. // @@ -278,6 +296,7 @@ class ChannelData { grpc_core::UniquePtr server_name_; grpc_core::UniquePtr target_uri_; channelz::ChannelNode* channelz_node_; + ChannelConfigHelper channel_config_helper_; // // Fields used in the data plane. Guarded by data_plane_mu. @@ -286,9 +305,11 @@ class ChannelData { std::unique_ptr picker_; QueuedPick* queued_picks_ = nullptr; // Linked list of queued picks. // Data from service config. + grpc_error* resolver_transient_failure_error_ = GRPC_ERROR_NONE; bool received_service_config_data_ = false; RefCountedPtr retry_throttle_data_; RefCountedPtr service_config_; + RefCountedPtr config_selector_; // // Fields used in the control plane. Guarded by work_serializer. @@ -300,6 +321,7 @@ class ChannelData { ConnectivityStateTracker state_tracker_; grpc_core::UniquePtr health_check_service_name_; RefCountedPtr saved_service_config_; + RefCountedPtr saved_config_selector_; bool received_first_resolver_result_ = false; // The number of SubchannelWrapper instances referencing a given Subchannel. std::map subchannel_refcount_map_; @@ -352,9 +374,6 @@ class CallData { RefCountedPtr subchannel_call() { return subchannel_call_; } - // Invoked by channel for queued picks once resolver results are available. - void MaybeApplyServiceConfigToCallLocked(grpc_call_element* elem); - // Invoked by channel for queued picks when the picker is updated. static void PickSubchannel(void* arg, grpc_error* error); @@ -742,13 +761,17 @@ class CallData { void CreateSubchannelCall(grpc_call_element* elem); // Invoked when a pick is completed, on both success or failure. static void PickDone(void* arg, grpc_error* error); - // Removes the call from the channel's list of queued picks. - void RemoveCallFromQueuedPicksLocked(grpc_call_element* elem); - // Adds the call to the channel's list of queued picks. - void AddCallToQueuedPicksLocked(grpc_call_element* elem); + // Removes the call from the channel's list of queued picks if present. + void MaybeRemoveCallFromQueuedPicksLocked(grpc_call_element* elem); + // Adds the call to the channel's list of queued picks if not already present. + void MaybeAddCallToQueuedPicksLocked(grpc_call_element* elem); // Applies service config to the call. Must be invoked once we know // that the resolver has returned results to the channel. - void ApplyServiceConfigToCallLocked(grpc_call_element* elem); + // If an error is returned, the error indicates the status with which + // the call should be failed. + grpc_error* ApplyServiceConfigToCallLocked( + grpc_call_element* elem, grpc_metadata_batch* initial_metadata); + void MaybeInvokeConfigSelectorCommitCallback(); // State for handling deadlines. // The code in deadline_filter.c requires this to be the first field. @@ -769,6 +792,7 @@ class CallData { RefCountedPtr retry_throttle_data_; const ClientChannelMethodParsedConfig* method_params_ = nullptr; std::map call_attributes_; + std::function on_call_committed_; RefCountedPtr subchannel_call_; @@ -1335,6 +1359,180 @@ class ChannelData::ClientChannelControlHelper ChannelData* chand_; }; +// +// ChannelData::ChannelConfigHelper +// + +// Synchronous callback from ResolvingLoadBalancingPolicy to process a +// resolver result update. +ChannelData::ChannelConfigHelper::ApplyServiceConfigResult +ChannelData::ChannelConfigHelper::ApplyServiceConfig( + const Resolver::Result& result) { + ApplyServiceConfigResult service_config_result; + RefCountedPtr service_config; + // If resolver did not return a service config or returned an invalid service + // config, we need a fallback service config. + if (result.service_config_error != GRPC_ERROR_NONE) { + // If the service config was invalid, then fallback to the saved service + // config. If there is no saved config either, use the default service + // config. + if (chand_->saved_service_config_ != nullptr) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) { + gpr_log(GPR_INFO, + "chand=%p: resolver returned invalid service config. " + "Continuing to use previous service config.", + chand_); + } + service_config = chand_->saved_service_config_; + } else if (chand_->default_service_config_ != nullptr) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) { + gpr_log(GPR_INFO, + "chand=%p: resolver returned invalid service config. Using " + "default service config provided by client API.", + chand_); + } + service_config = chand_->default_service_config_; + } + } else if (result.service_config == nullptr) { + if (chand_->default_service_config_ != nullptr) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) { + gpr_log(GPR_INFO, + "chand=%p: resolver returned no service config. Using default " + "service config provided by client API.", + chand_); + } + service_config = chand_->default_service_config_; + } + } else { + service_config = result.service_config; + } + service_config_result.service_config_error = + GRPC_ERROR_REF(result.service_config_error); + if (service_config == nullptr && + result.service_config_error != GRPC_ERROR_NONE) { + service_config_result.no_valid_service_config = true; + return service_config_result; + } + // Process service config. + grpc_core::UniquePtr service_config_json; + const internal::ClientChannelGlobalParsedConfig* parsed_service_config = + nullptr; + if (service_config != nullptr) { + parsed_service_config = + static_cast( + service_config->GetGlobalParsedConfig( + internal::ClientChannelServiceConfigParser::ParserIndex())); + } + // Check if the config has changed. + service_config_result.service_config_changed = + ((service_config == nullptr) != + (chand_->saved_service_config_ == nullptr)) || + (service_config != nullptr && + service_config->json_string() != + chand_->saved_service_config_->json_string()); + if (service_config_result.service_config_changed) { + service_config_json.reset(gpr_strdup( + service_config != nullptr ? service_config->json_string().c_str() + : "")); + if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) { + gpr_log(GPR_INFO, + "chand=%p: resolver returned updated service config: \"%s\"", + chand_, service_config_json.get()); + } + // Save health check service name. + if (service_config != nullptr) { + chand_->health_check_service_name_.reset( + gpr_strdup(parsed_service_config->health_check_service_name())); + } else { + chand_->health_check_service_name_.reset(); + } + // Update health check service name used by existing subchannel wrappers. + for (auto* subchannel_wrapper : chand_->subchannel_wrappers_) { + subchannel_wrapper->UpdateHealthCheckServiceName( + grpc_core::UniquePtr( + gpr_strdup(chand_->health_check_service_name_.get()))); + } + // Save service config. + chand_->saved_service_config_ = std::move(service_config); + } + // Find LB policy config. + ProcessLbPolicy(result, parsed_service_config, + &service_config_result.lb_policy_config); + grpc_core::UniquePtr lb_policy_name( + gpr_strdup((service_config_result.lb_policy_config)->name())); + // Swap out the data used by GetChannelInfo(). + { + MutexLock lock(&chand_->info_mu_); + chand_->info_lb_policy_name_ = std::move(lb_policy_name); + if (service_config_json != nullptr) { + chand_->info_service_config_json_ = std::move(service_config_json); + } + } + // Return results. + return service_config_result; +} + +void ChannelData::ChannelConfigHelper::ApplyConfigSelector( + bool service_config_changed, + RefCountedPtr config_selector) { + chand_->UpdateServiceConfigInDataPlaneLocked(service_config_changed, + std::move(config_selector)); +} + +void ChannelData::ChannelConfigHelper::ResolverTransientFailure( + grpc_error* error) { + MutexLock lock(&chand_->data_plane_mu_); + GRPC_ERROR_UNREF(chand_->resolver_transient_failure_error_); + chand_->resolver_transient_failure_error_ = error; +} + +void ChannelData::ChannelConfigHelper::ProcessLbPolicy( + const Resolver::Result& resolver_result, + const internal::ClientChannelGlobalParsedConfig* parsed_service_config, + RefCountedPtr* lb_policy_config) { + // Prefer the LB policy config found in the service config. + if (parsed_service_config != nullptr && + parsed_service_config->parsed_lb_config() != nullptr) { + *lb_policy_config = parsed_service_config->parsed_lb_config(); + return; + } + // Try the deprecated LB policy name from the service config. + // If not, try the setting from channel args. + const char* policy_name = nullptr; + if (parsed_service_config != nullptr && + !parsed_service_config->parsed_deprecated_lb_policy().empty()) { + policy_name = parsed_service_config->parsed_deprecated_lb_policy().c_str(); + } else { + const grpc_arg* channel_arg = + grpc_channel_args_find(resolver_result.args, GRPC_ARG_LB_POLICY_NAME); + policy_name = grpc_channel_arg_get_string(channel_arg); + } + // Use pick_first if nothing was specified and we didn't select grpclb + // above. + if (policy_name == nullptr) policy_name = "pick_first"; + // Now that we have the policy name, construct an empty config for it. + Json config_json = Json::Array{Json::Object{ + {policy_name, Json::Object{}}, + }}; + grpc_error* parse_error = GRPC_ERROR_NONE; + *lb_policy_config = LoadBalancingPolicyRegistry::ParseLoadBalancingConfig( + config_json, &parse_error); + // The policy name came from one of three places: + // - The deprecated loadBalancingPolicy field in the service config, + // in which case the code in ClientChannelServiceConfigParser + // already verified that the policy does not require a config. + // - One of the hard-coded values here, all of which are known to not + // require a config. + // - A channel arg, in which case the application did something that + // is a misuse of our API. + // In the first two cases, these assertions will always be true. In + // the last case, this is probably fine for now. + // TODO(roth): If the last case becomes a problem, add better error + // handling here. + GPR_ASSERT(*lb_policy_config != nullptr); + GPR_ASSERT(parse_error == GRPC_ERROR_NONE); +} + // // ChannelData implementation // @@ -1393,6 +1591,7 @@ ChannelData::ChannelData(grpc_channel_element_args* args, grpc_error** error) client_channel_factory_( ClientChannelFactory::GetFromChannelArgs(args->channel_args)), channelz_node_(GetChannelzNode(args->channel_args)), + channel_config_helper_(this), work_serializer_(std::make_shared()), interested_parties_(grpc_pollset_set_create()), subchannel_pool_(GetSubchannelPool(args->channel_args)), @@ -1461,6 +1660,7 @@ ChannelData::~ChannelData() { } DestroyResolvingLoadBalancingPolicyLocked(); grpc_channel_args_destroy(channel_args_); + GRPC_ERROR_UNREF(resolver_transient_failure_error_); // Stop backup polling. grpc_client_channel_stop_backup_polling(interested_parties_); grpc_pollset_set_destroy(interested_parties_); @@ -1475,6 +1675,7 @@ void ChannelData::UpdateStateAndPickerLocked( if (picker_ == nullptr) { health_check_service_name_.reset(); saved_service_config_.reset(); + saved_config_selector_.reset(); received_first_resolver_result_ = false; } // Update connectivity state. @@ -1497,9 +1698,11 @@ void ChannelData::UpdateStateAndPickerLocked( // - refs to subchannel wrappers in the keys of pending_subchannel_updates_ // - ref stored in retry_throttle_data_ // - ref stored in service_config_ + // - ref stored in config_selector_ // - ownership of the existing picker in picker_ RefCountedPtr retry_throttle_data_to_unref; RefCountedPtr service_config_to_unref; + RefCountedPtr config_selector_to_unref; { MutexLock lock(&data_plane_mu_); // Handle subchannel updates. @@ -1524,6 +1727,7 @@ void ChannelData::UpdateStateAndPickerLocked( // Note: We save the objects to unref until after the lock is released. retry_throttle_data_to_unref = std::move(retry_throttle_data_); service_config_to_unref = std::move(service_config_); + config_selector_to_unref = std::move(config_selector_); } // Re-process queued picks. for (QueuedPick* pick = queued_picks_; pick != nullptr; pick = pick->next) { @@ -1540,24 +1744,72 @@ void ChannelData::UpdateStateAndPickerLocked( pending_subchannel_updates_.clear(); } -void ChannelData::UpdateServiceConfigLocked( - RefCountedPtr retry_throttle_data, - RefCountedPtr service_config) { +void ChannelData::UpdateServiceConfigInDataPlaneLocked( + bool service_config_changed, + RefCountedPtr config_selector) { + // Check if ConfigSelector has changed. + const bool config_selector_changed = + saved_config_selector_ != config_selector; + saved_config_selector_ = config_selector; + // We want to set the service config at least once, even if the + // resolver does not return a config, because that ensures that we + // disable retries if they are not enabled in the service config. + // TODO(roth): Consider removing the received_first_resolver_result_ check + // when we implement transparent retries. + if (!service_config_changed && !config_selector_changed && + received_first_resolver_result_) { + return; + } + received_first_resolver_result_ = true; + // Get retry throttle data from service config. + RefCountedPtr retry_throttle_data; + if (saved_service_config_ != nullptr) { + const internal::ClientChannelGlobalParsedConfig* parsed_service_config = + static_cast( + saved_service_config_->GetGlobalParsedConfig( + internal::ClientChannelServiceConfigParser::ParserIndex())); + if (parsed_service_config != nullptr) { + absl::optional + retry_throttle_config = parsed_service_config->retry_throttling(); + if (retry_throttle_config.has_value()) { + retry_throttle_data = + internal::ServerRetryThrottleMap::GetDataForServer( + server_name_.get(), + retry_throttle_config.value().max_milli_tokens, + retry_throttle_config.value().milli_token_ratio); + } + } + } + // Create default config selector if not provided by resolver. + if (config_selector == nullptr) { + config_selector = + MakeRefCounted(saved_service_config_); + } // Grab data plane lock to update service config. // // We defer unreffing the old values (and deallocating memory) until // after releasing the lock to keep the critical section small. + RefCountedPtr service_config_to_unref = saved_service_config_; + RefCountedPtr config_selector_to_unref = + std::move(config_selector); { MutexLock lock(&data_plane_mu_); + GRPC_ERROR_UNREF(resolver_transient_failure_error_); + resolver_transient_failure_error_ = GRPC_ERROR_NONE; // Update service config. received_service_config_data_ = true; // Old values will be unreffed after lock is released. retry_throttle_data_.swap(retry_throttle_data); - service_config_.swap(service_config); - // Apply service config to queued picks. + service_config_.swap(service_config_to_unref); + config_selector_.swap(config_selector_to_unref); + // Re-process queued picks. for (QueuedPick* pick = queued_picks_; pick != nullptr; pick = pick->next) { - CallData* calld = static_cast(pick->elem->call_data); - calld->MaybeApplyServiceConfigToCallLocked(pick->elem); + grpc_call_element* elem = pick->elem; + CallData* calld = static_cast(elem->call_data); + grpc_error* error = GRPC_ERROR_NONE; + if (calld->PickSubchannelLocked(elem, &error)) { + calld->AsyncPickDone(elem, error); + } } } // Old values will be unreffed after lock is released when they go out @@ -1574,7 +1826,7 @@ void ChannelData::CreateResolvingLoadBalancingPolicyLocked() { grpc_core::UniquePtr target_uri(gpr_strdup(target_uri_.get())); resolving_lb_policy_.reset(new ResolvingLoadBalancingPolicy( std::move(lb_args), &grpc_client_channel_routing_trace, - std::move(target_uri), ProcessResolverResultLocked, this)); + std::move(target_uri), &channel_config_helper_)); grpc_pollset_set_add_pollset_set(resolving_lb_policy_->interested_parties(), interested_parties_); if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) { @@ -1591,180 +1843,6 @@ void ChannelData::DestroyResolvingLoadBalancingPolicyLocked() { } } -void ChannelData::ProcessLbPolicy( - const Resolver::Result& resolver_result, - const internal::ClientChannelGlobalParsedConfig* parsed_service_config, - RefCountedPtr* lb_policy_config) { - // Prefer the LB policy config found in the service config. - if (parsed_service_config != nullptr && - parsed_service_config->parsed_lb_config() != nullptr) { - *lb_policy_config = parsed_service_config->parsed_lb_config(); - return; - } - // Try the deprecated LB policy name from the service config. - // If not, try the setting from channel args. - const char* policy_name = nullptr; - if (parsed_service_config != nullptr && - !parsed_service_config->parsed_deprecated_lb_policy().empty()) { - policy_name = parsed_service_config->parsed_deprecated_lb_policy().c_str(); - } else { - const grpc_arg* channel_arg = - grpc_channel_args_find(resolver_result.args, GRPC_ARG_LB_POLICY_NAME); - policy_name = grpc_channel_arg_get_string(channel_arg); - } - // Use pick_first if nothing was specified and we didn't select grpclb - // above. - if (policy_name == nullptr) policy_name = "pick_first"; - // Now that we have the policy name, construct an empty config for it. - Json config_json = Json::Array{Json::Object{ - {policy_name, Json::Object{}}, - }}; - grpc_error* parse_error = GRPC_ERROR_NONE; - *lb_policy_config = LoadBalancingPolicyRegistry::ParseLoadBalancingConfig( - config_json, &parse_error); - // The policy name came from one of three places: - // - The deprecated loadBalancingPolicy field in the service config, - // in which case the code in ClientChannelServiceConfigParser - // already verified that the policy does not require a config. - // - One of the hard-coded values here, all of which are known to not - // require a config. - // - A channel arg, in which case the application did something that - // is a misuse of our API. - // In the first two cases, these assertions will always be true. In - // the last case, this is probably fine for now. - // TODO(roth): If the last case becomes a problem, add better error - // handling here. - GPR_ASSERT(*lb_policy_config != nullptr); - GPR_ASSERT(parse_error == GRPC_ERROR_NONE); -} - -// Synchronous callback from ResolvingLoadBalancingPolicy to process a -// resolver result update. -bool ChannelData::ProcessResolverResultLocked( - void* arg, const Resolver::Result& result, - RefCountedPtr* lb_policy_config, - grpc_error** service_config_error, bool* no_valid_service_config) { - ChannelData* chand = static_cast(arg); - RefCountedPtr service_config; - // If resolver did not return a service config or returned an invalid service - // config, we need a fallback service config. - if (result.service_config_error != GRPC_ERROR_NONE) { - // If the service config was invalid, then fallback to the saved service - // config. If there is no saved config either, use the default service - // config. - if (chand->saved_service_config_ != nullptr) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) { - gpr_log(GPR_INFO, - "chand=%p: resolver returned invalid service config. " - "Continuing to use previous service config.", - chand); - } - service_config = chand->saved_service_config_; - } else if (chand->default_service_config_ != nullptr) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) { - gpr_log(GPR_INFO, - "chand=%p: resolver returned invalid service config. Using " - "default service config provided by client API.", - chand); - } - service_config = chand->default_service_config_; - } - } else if (result.service_config == nullptr) { - if (chand->default_service_config_ != nullptr) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) { - gpr_log(GPR_INFO, - "chand=%p: resolver returned no service config. Using default " - "service config provided by client API.", - chand); - } - service_config = chand->default_service_config_; - } - } else { - service_config = result.service_config; - } - *service_config_error = GRPC_ERROR_REF(result.service_config_error); - if (service_config == nullptr && - result.service_config_error != GRPC_ERROR_NONE) { - *no_valid_service_config = true; - return false; - } - // Process service config. - grpc_core::UniquePtr service_config_json; - const internal::ClientChannelGlobalParsedConfig* parsed_service_config = - nullptr; - if (service_config != nullptr) { - parsed_service_config = - static_cast( - service_config->GetGlobalParsedConfig( - internal::ClientChannelServiceConfigParser::ParserIndex())); - } - // Check if the config has changed. - const bool service_config_changed = - ((service_config == nullptr) != - (chand->saved_service_config_ == nullptr)) || - (service_config != nullptr && - service_config->json_string() != - chand->saved_service_config_->json_string()); - if (service_config_changed) { - service_config_json.reset(gpr_strdup( - service_config != nullptr ? service_config->json_string().c_str() - : "")); - if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) { - gpr_log(GPR_INFO, - "chand=%p: resolver returned updated service config: \"%s\"", - chand, service_config_json.get()); - } - // Save health check service name. - if (service_config != nullptr) { - chand->health_check_service_name_.reset( - gpr_strdup(parsed_service_config->health_check_service_name())); - } else { - chand->health_check_service_name_.reset(); - } - // Update health check service name used by existing subchannel wrappers. - for (auto* subchannel_wrapper : chand->subchannel_wrappers_) { - subchannel_wrapper->UpdateHealthCheckServiceName( - grpc_core::UniquePtr( - gpr_strdup(chand->health_check_service_name_.get()))); - } - // Save service config. - chand->saved_service_config_ = std::move(service_config); - } - // We want to set the service config at least once. This should not really be - // needed, but we are doing it as a defensive approach. This can be removed, - // if we feel it is unnecessary. - if (service_config_changed || !chand->received_first_resolver_result_) { - chand->received_first_resolver_result_ = true; - RefCountedPtr retry_throttle_data; - if (parsed_service_config != nullptr) { - absl::optional - retry_throttle_config = parsed_service_config->retry_throttling(); - if (retry_throttle_config.has_value()) { - retry_throttle_data = - internal::ServerRetryThrottleMap::GetDataForServer( - chand->server_name_.get(), - retry_throttle_config.value().max_milli_tokens, - retry_throttle_config.value().milli_token_ratio); - } - } - chand->UpdateServiceConfigLocked(std::move(retry_throttle_data), - chand->saved_service_config_); - } - chand->ProcessLbPolicy(result, parsed_service_config, lb_policy_config); - grpc_core::UniquePtr lb_policy_name( - gpr_strdup((*lb_policy_config)->name())); - // Swap out the data used by GetChannelInfo(). - { - MutexLock lock(&chand->info_mu_); - chand->info_lb_policy_name_ = std::move(lb_policy_name); - if (service_config_json != nullptr) { - chand->info_service_config_json_ = std::move(service_config_json); - } - } - // Return results. - return service_config_changed; -} - grpc_error* ChannelData::DoPingLocked(grpc_transport_op* op) { if (state_tracker_.state() != GRPC_CHANNEL_READY) { return GRPC_ERROR_CREATE_FROM_STATIC_STRING("channel not connected"); @@ -2807,6 +2885,7 @@ void CallData::RecvInitialMetadataReady(void* arg, grpc_error* error) { } // Received valid initial metadata, so commit the call. calld->RetryCommit(elem, retry_state); + calld->MaybeInvokeConfigSelectorCommitCallback(); // Invoke the callback to return the result to the surface. // Manually invoking a callback function; it does not take ownership of error. calld->InvokeRecvInitialMetadataCallback(batch_data, error); @@ -2893,6 +2972,7 @@ void CallData::RecvMessageReady(void* arg, grpc_error* error) { } // Received a valid message, so commit the call. calld->RetryCommit(elem, retry_state); + calld->MaybeInvokeConfigSelectorCommitCallback(); // Invoke the callback to return the result to the surface. // Manually invoking a callback function; it does not take ownership of error. calld->InvokeRecvMessageCallback(batch_data, error); @@ -3094,6 +3174,7 @@ void CallData::RecvTrailingMetadataReady(void* arg, grpc_error* error) { } // Not retrying, so commit the call. calld->RetryCommit(elem, retry_state); + calld->MaybeInvokeConfigSelectorCommitCallback(); // Run any necessary closures. calld->RunClosuresForCompletedCall(batch_data, GRPC_ERROR_REF(error)); } @@ -3716,7 +3797,7 @@ class CallData::QueuedPickCanceller { } if (calld->pick_canceller_ == self && error != GRPC_ERROR_NONE) { // Remove pick from list of queued picks. - calld->RemoveCallFromQueuedPicksLocked(self->elem_); + calld->MaybeRemoveCallFromQueuedPicksLocked(self->elem_); // Fail pending batches on the call. calld->PendingBatchesFail(self->elem_, GRPC_ERROR_REF(error), YieldCallCombinerIfPendingBatchesFound); @@ -3729,7 +3810,8 @@ class CallData::QueuedPickCanceller { grpc_closure closure_; }; -void CallData::RemoveCallFromQueuedPicksLocked(grpc_call_element* elem) { +void CallData::MaybeRemoveCallFromQueuedPicksLocked(grpc_call_element* elem) { + if (!pick_queued_) return; auto* chand = static_cast(elem->channel_data); if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) { gpr_log(GPR_INFO, "chand=%p calld=%p: removing from queued picks list", @@ -3741,7 +3823,8 @@ void CallData::RemoveCallFromQueuedPicksLocked(grpc_call_element* elem) { pick_canceller_ = nullptr; } -void CallData::AddCallToQueuedPicksLocked(grpc_call_element* elem) { +void CallData::MaybeAddCallToQueuedPicksLocked(grpc_call_element* elem) { + if (pick_queued_) return; auto* chand = static_cast(elem->channel_data); if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) { gpr_log(GPR_INFO, "chand=%p calld=%p: adding to queued picks list", chand, @@ -3754,23 +3837,29 @@ void CallData::AddCallToQueuedPicksLocked(grpc_call_element* elem) { pick_canceller_ = new QueuedPickCanceller(elem); } -void CallData::ApplyServiceConfigToCallLocked(grpc_call_element* elem) { +grpc_error* CallData::ApplyServiceConfigToCallLocked( + grpc_call_element* elem, grpc_metadata_batch* initial_metadata) { ChannelData* chand = static_cast(elem->channel_data); if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) { gpr_log(GPR_INFO, "chand=%p calld=%p: applying service config to call", chand, this); } + ConfigSelector* config_selector = chand->config_selector(); auto service_config = chand->service_config(); if (service_config != nullptr) { + // Use the ConfigSelector to determine the config for the call. + ConfigSelector::CallConfig call_config = + config_selector->GetCallConfig({&path_, initial_metadata}); + if (call_config.error != GRPC_ERROR_NONE) return call_config.error; + call_attributes_ = std::move(call_config.call_attributes); + on_call_committed_ = std::move(call_config.on_call_committed); // Create a ServiceConfigCallData for the call. This stores a ref to the // ServiceConfig and caches the right set of parsed configs to use for // the call. The MethodConfig will store itself in the call context, // so that it can be accessed by filters in the subchannel, and it // will be cleaned up when the call ends. - const auto* method_params_vector = - service_config->GetMethodParsedConfigVector(path_); auto* service_config_call_data = arena_->New( - std::move(service_config), method_params_vector, call_context_); + std::move(service_config), call_config.method_configs, call_context_); // Apply our own method params to the call. method_params_ = static_cast( service_config_call_data->GetMethodParsedConfig( @@ -3812,16 +3901,13 @@ void CallData::ApplyServiceConfigToCallLocked(grpc_call_element* elem) { if (method_params_ == nullptr || method_params_->retry_policy() == nullptr) { enable_retries_ = false; } + return GRPC_ERROR_NONE; } -void CallData::MaybeApplyServiceConfigToCallLocked(grpc_call_element* elem) { - ChannelData* chand = static_cast(elem->channel_data); - // Apply service config data to the call only once, and only if the - // channel has the data available. - if (GPR_LIKELY(chand->received_service_config_data() && - !service_config_applied_)) { - service_config_applied_ = true; - ApplyServiceConfigToCallLocked(elem); +void CallData::MaybeInvokeConfigSelectorCommitCallback() { + if (on_call_committed_ != nullptr) { + on_call_committed_(); + on_call_committed_ = nullptr; } } @@ -3882,11 +3968,45 @@ bool CallData::PickSubchannelLocked(grpc_call_element* elem, GRPC_ERROR_NONE); // Queue the pick, so that it will be attempted once the channel // becomes connected. - AddCallToQueuedPicksLocked(elem); + MaybeAddCallToQueuedPicksLocked(elem); + return false; + } + grpc_metadata_batch* initial_metadata_batch = + seen_send_initial_metadata_ + ? &send_initial_metadata_ + : pending_batches_[0] + .batch->payload->send_initial_metadata.send_initial_metadata; + // Grab initial metadata flags so that we can check later if the call has + // wait_for_ready enabled. + const uint32_t send_initial_metadata_flags = + seen_send_initial_metadata_ ? send_initial_metadata_flags_ + : pending_batches_[0] + .batch->payload->send_initial_metadata + .send_initial_metadata_flags; + // Avoid picking if we haven't yet received service config data. + if (GPR_UNLIKELY(!chand->received_service_config_data())) { + // If the resolver returned transient failure before returning the + // first service config, fail any non-wait_for_ready calls. + grpc_error* resolver_error = chand->resolver_transient_failure_error(); + if (resolver_error != GRPC_ERROR_NONE && + (send_initial_metadata_flags & GRPC_INITIAL_METADATA_WAIT_FOR_READY) == + 0) { + MaybeRemoveCallFromQueuedPicksLocked(elem); + *error = GRPC_ERROR_REF(resolver_error); + return true; + } + // Either the resolver has not yet returned a result, or it has + // returned transient failure but the call is wait_for_ready. In + // either case, queue the call. + MaybeAddCallToQueuedPicksLocked(elem); return false; } - // Apply service config to call if needed. - MaybeApplyServiceConfigToCallLocked(elem); + // Apply service config to call if not yet applied. + if (GPR_LIKELY(!service_config_applied_)) { + service_config_applied_ = true; + *error = ApplyServiceConfigToCallLocked(elem, initial_metadata_batch); + if (*error != GRPC_ERROR_NONE) return true; + } // If this is a retry, use the send_initial_metadata payload that // we've cached; otherwise, use the pending batch. The // send_initial_metadata batch will be the first pending batch in the @@ -3899,20 +4019,8 @@ bool CallData::PickSubchannelLocked(grpc_call_element* elem, // attempt) to the LB policy instead the one from the parent channel. LoadBalancingPolicy::PickArgs pick_args; pick_args.call_state = &lb_call_state_; - Metadata initial_metadata( - this, - seen_send_initial_metadata_ - ? &send_initial_metadata_ - : pending_batches_[0] - .batch->payload->send_initial_metadata.send_initial_metadata); + Metadata initial_metadata(this, initial_metadata_batch); pick_args.initial_metadata = &initial_metadata; - // Grab initial metadata flags so that we can check later if the call has - // wait_for_ready enabled. - const uint32_t send_initial_metadata_flags = - seen_send_initial_metadata_ ? send_initial_metadata_flags_ - : pending_batches_[0] - .batch->payload->send_initial_metadata - .send_initial_metadata_flags; // Attempt pick. auto result = chand->picker()->Pick(pick_args); if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) { @@ -3927,7 +4035,8 @@ bool CallData::PickSubchannelLocked(grpc_call_element* elem, grpc_error* disconnect_error = chand->disconnect_error(); if (disconnect_error != GRPC_ERROR_NONE) { GRPC_ERROR_UNREF(result.error); - if (pick_queued_) RemoveCallFromQueuedPicksLocked(elem); + MaybeRemoveCallFromQueuedPicksLocked(elem); + MaybeInvokeConfigSelectorCommitCallback(); *error = GRPC_ERROR_REF(disconnect_error); return true; } @@ -3948,8 +4057,9 @@ bool CallData::PickSubchannelLocked(grpc_call_element* elem, "Failed to pick subchannel", &result.error, 1); GRPC_ERROR_UNREF(result.error); *error = new_error; + MaybeInvokeConfigSelectorCommitCallback(); } - if (pick_queued_) RemoveCallFromQueuedPicksLocked(elem); + MaybeRemoveCallFromQueuedPicksLocked(elem); return !retried; } // If wait_for_ready is true, then queue to retry when we get a new @@ -3958,22 +4068,24 @@ bool CallData::PickSubchannelLocked(grpc_call_element* elem, } // Fallthrough case LoadBalancingPolicy::PickResult::PICK_QUEUE: - if (!pick_queued_) AddCallToQueuedPicksLocked(elem); + MaybeAddCallToQueuedPicksLocked(elem); return false; default: // PICK_COMPLETE - if (pick_queued_) RemoveCallFromQueuedPicksLocked(elem); + MaybeRemoveCallFromQueuedPicksLocked(elem); // Handle drops. if (GPR_UNLIKELY(result.subchannel == nullptr)) { result.error = grpc_error_set_int( GRPC_ERROR_CREATE_FROM_STATIC_STRING( "Call dropped by load balancing policy"), GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE); + MaybeInvokeConfigSelectorCommitCallback(); } else { // Grab a ref to the connected subchannel while we're still // holding the data plane mutex. connected_subchannel_ = chand->GetConnectedSubchannelInDataPlane(result.subchannel.get()); GPR_ASSERT(connected_subchannel_ != nullptr); + if (retry_committed_) MaybeInvokeConfigSelectorCommitCallback(); } lb_recv_trailing_metadata_ready_ = result.recv_trailing_metadata_ready; *error = result.error; diff --git a/src/core/ext/filters/client_channel/config_selector.cc b/src/core/ext/filters/client_channel/config_selector.cc new file mode 100644 index 00000000000..e5d2a3f0be3 --- /dev/null +++ b/src/core/ext/filters/client_channel/config_selector.cc @@ -0,0 +1,62 @@ +// +// Copyright 2020 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include + +#include "src/core/ext/filters/client_channel/config_selector.h" + +#include "src/core/lib/channel/channel_args.h" + +// Channel arg key for ConfigSelector. +#define GRPC_ARG_CONFIG_SELECTOR "grpc.internal.config_selector" + +namespace grpc_core { + +namespace { + +void* ConfigSelectorArgCopy(void* p) { + ConfigSelector* config_selector = static_cast(p); + config_selector->Ref().release(); + return p; +} + +void ConfigSelectorArgDestroy(void* p) { + ConfigSelector* config_selector = static_cast(p); + config_selector->Unref(); +} + +int ConfigSelectorArgCmp(void* p, void* q) { return GPR_ICMP(p, q); } + +const grpc_arg_pointer_vtable kChannelArgVtable = { + ConfigSelectorArgCopy, ConfigSelectorArgDestroy, ConfigSelectorArgCmp}; + +} // namespace + +grpc_arg ConfigSelector::MakeChannelArg() const { + return grpc_channel_arg_pointer_create( + const_cast(GRPC_ARG_CONFIG_SELECTOR), + const_cast(this), &kChannelArgVtable); +} + +RefCountedPtr ConfigSelector::GetFromChannelArgs( + const grpc_channel_args& args) { + ConfigSelector* config_selector = + grpc_channel_args_find_pointer(&args, + GRPC_ARG_CONFIG_SELECTOR); + return config_selector != nullptr ? config_selector->Ref() : nullptr; +} + +} // namespace grpc_core diff --git a/src/core/ext/filters/client_channel/config_selector.h b/src/core/ext/filters/client_channel/config_selector.h new file mode 100644 index 00000000000..efd27341fa5 --- /dev/null +++ b/src/core/ext/filters/client_channel/config_selector.h @@ -0,0 +1,91 @@ +// +// Copyright 2020 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#ifndef GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_CONFIG_SELECTOR_H +#define GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_CONFIG_SELECTOR_H + +#include + +#include +#include + +#include "absl/strings/string_view.h" + +#include +#include + +#include "src/core/ext/filters/client_channel/service_config.h" +#include "src/core/ext/filters/client_channel/service_config_parser.h" +#include "src/core/lib/gprpp/ref_counted.h" +#include "src/core/lib/gprpp/ref_counted_ptr.h" +#include "src/core/lib/transport/metadata_batch.h" + +namespace grpc_core { + +// Internal API used to allow resolver implementations to override +// MethodConfig and provide input to LB policies on a per-call basis. +class ConfigSelector : public RefCounted { + public: + struct GetCallConfigArgs { + grpc_slice* path; + grpc_metadata_batch* initial_metadata; + }; + + struct CallConfig { + // Can be set to indicate the call should be failed. + grpc_error* error = GRPC_ERROR_NONE; + // The per-method parsed configs that will be passed to + // ServiceConfigCallData. + const ServiceConfigParser::ParsedConfigVector* method_configs = nullptr; + // Call attributes that will be accessible to LB policy implementations. + std::map call_attributes; + // A callback that, if set, will be invoked when the call is + // committed (i.e., when we know that we will never again need to + // ask the picker for a subchannel for this call). + std::function on_call_committed; + }; + + virtual ~ConfigSelector() = default; + + virtual CallConfig GetCallConfig(GetCallConfigArgs args) = 0; + + grpc_arg MakeChannelArg() const; + static RefCountedPtr GetFromChannelArgs( + const grpc_channel_args& args); +}; + +// Default ConfigSelector that gets the MethodConfig from the service config. +class DefaultConfigSelector : public ConfigSelector { + public: + explicit DefaultConfigSelector(RefCountedPtr service_config) + : service_config_(std::move(service_config)) {} + + CallConfig GetCallConfig(GetCallConfigArgs args) override { + CallConfig call_config; + if (service_config_ != nullptr) { + call_config.method_configs = + service_config_->GetMethodParsedConfigVector(*args.path); + } + return call_config; + } + + private: + RefCountedPtr service_config_; +}; + +} // namespace grpc_core + +#endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_CONFIG_SELECTOR_H */ diff --git a/src/core/ext/filters/client_channel/resolving_lb_policy.cc b/src/core/ext/filters/client_channel/resolving_lb_policy.cc index 74a5c29968b..356123c50ae 100644 --- a/src/core/ext/filters/client_channel/resolving_lb_policy.cc +++ b/src/core/ext/filters/client_channel/resolving_lb_policy.cc @@ -145,14 +145,12 @@ class ResolvingLoadBalancingPolicy::ResolvingControlHelper ResolvingLoadBalancingPolicy::ResolvingLoadBalancingPolicy( Args args, TraceFlag* tracer, grpc_core::UniquePtr target_uri, - ProcessResolverResultCallback process_resolver_result, - void* process_resolver_result_user_data) + ChannelConfigHelper* helper) : LoadBalancingPolicy(std::move(args)), tracer_(tracer), target_uri_(std::move(target_uri)), - process_resolver_result_(process_resolver_result), - process_resolver_result_user_data_(process_resolver_result_user_data) { - GPR_ASSERT(process_resolver_result != nullptr); + helper_(helper) { + GPR_ASSERT(helper_ != nullptr); resolver_ = ResolverRegistry::CreateResolver( target_uri_.get(), args.args, interested_parties(), work_serializer(), absl::make_unique(Ref())); @@ -214,6 +212,7 @@ void ResolvingLoadBalancingPolicy::OnResolverError(grpc_error* error) { if (lb_policy_ == nullptr) { grpc_error* state_error = GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING( "Resolver transient failure", &error, 1); + helper_->ResolverTransientFailure(GRPC_ERROR_REF(state_error)); channel_control_helper()->UpdateState( GRPC_CHANNEL_TRANSIENT_FAILURE, absl::make_unique(state_error)); @@ -304,45 +303,51 @@ void ResolvingLoadBalancingPolicy::OnResolverResultChangedLocked( TraceStringVector trace_strings; const bool resolution_contains_addresses = result.addresses.size() > 0; // Process the resolver result. - RefCountedPtr lb_policy_config; - bool service_config_changed = false; - std::string service_config_error_string; - if (process_resolver_result_ != nullptr) { - grpc_error* service_config_error = GRPC_ERROR_NONE; - bool no_valid_service_config = false; - service_config_changed = process_resolver_result_( - process_resolver_result_user_data_, result, &lb_policy_config, - &service_config_error, &no_valid_service_config); - if (service_config_error != GRPC_ERROR_NONE) { - service_config_error_string = grpc_error_string(service_config_error); - if (no_valid_service_config) { + ChannelConfigHelper::ApplyServiceConfigResult service_config_result; + if (helper_ != nullptr) { + service_config_result = helper_->ApplyServiceConfig(result); + if (service_config_result.service_config_error != GRPC_ERROR_NONE) { + if (service_config_result.no_valid_service_config) { // We received an invalid service config and we don't have a // fallback service config. - OnResolverError(service_config_error); - } else { - GRPC_ERROR_UNREF(service_config_error); + OnResolverError(service_config_result.service_config_error); + service_config_result.service_config_error = GRPC_ERROR_NONE; } } } else { - lb_policy_config = child_lb_config_; + service_config_result.lb_policy_config = child_lb_config_; } - if (lb_policy_config != nullptr) { - // Create or update LB policy, as needed. - CreateOrUpdateLbPolicyLocked(std::move(lb_policy_config), - std::move(result)); + // Before we send the args to the LB policy, grab the ConfigSelector for + // later use. + RefCountedPtr config_selector = + ConfigSelector::GetFromChannelArgs(*result.args); + // Create or update LB policy, as needed. + if (service_config_result.lb_policy_config != nullptr) { + CreateOrUpdateLbPolicyLocked( + std::move(service_config_result.lb_policy_config), std::move(result)); + } + // Apply ConfigSelector to channel. + // This needs to happen after the LB policy has been updated, since + // the ConfigSelector may need the LB policy to know about new + // destinations before it can send RPCs to those destinations. + if (helper_ != nullptr) { + helper_->ApplyConfigSelector(service_config_result.service_config_changed, + std::move(config_selector)); } // Add channel trace event. - if (service_config_changed) { + if (service_config_result.service_config_changed) { // TODO(ncteisen): might be worth somehow including a snippet of the // config in the trace, at the risk of bloating the trace logs. trace_strings.push_back("Service config changed"); } - if (!service_config_error_string.empty()) { - trace_strings.push_back(service_config_error_string.c_str()); + if (service_config_result.service_config_error != GRPC_ERROR_NONE) { + trace_strings.push_back( + grpc_error_string(service_config_result.service_config_error)); } MaybeAddTraceMessagesForAddressChangesLocked(resolution_contains_addresses, &trace_strings); ConcatenateAndAddChannelTraceLocked(trace_strings); + GRPC_ERROR_UNREF(service_config_result.service_config_error); } } // namespace grpc_core diff --git a/src/core/ext/filters/client_channel/resolving_lb_policy.h b/src/core/ext/filters/client_channel/resolving_lb_policy.h index 39815e28039..98ea37fbb64 100644 --- a/src/core/ext/filters/client_channel/resolving_lb_policy.h +++ b/src/core/ext/filters/client_channel/resolving_lb_policy.h @@ -23,6 +23,7 @@ #include "absl/container/inlined_vector.h" +#include "src/core/ext/filters/client_channel/config_selector.h" #include "src/core/ext/filters/client_channel/lb_policy.h" #include "src/core/ext/filters/client_channel/lb_policy_factory.h" #include "src/core/ext/filters/client_channel/resolver.h" @@ -52,22 +53,37 @@ namespace grpc_core { // child LB policy and config to use. class ResolvingLoadBalancingPolicy : public LoadBalancingPolicy { public: - // Synchronous callback that takes the resolver result and sets - // lb_policy_config to point to the right data. - // Returns true if the service config has changed since the last result. - // If the returned no_valid_service_config is true, that means that we - // don't have a valid service config to use, and we should set the channel - // to be in TRANSIENT_FAILURE. - typedef bool (*ProcessResolverResultCallback)( - void* user_data, const Resolver::Result& result, - RefCountedPtr* lb_policy_config, - grpc_error** service_config_error, bool* no_valid_service_config); - // If error is set when this returns, then construction failed, and - // the caller may not use the new object. - ResolvingLoadBalancingPolicy( - Args args, TraceFlag* tracer, grpc_core::UniquePtr target_uri, - ProcessResolverResultCallback process_resolver_result, - void* process_resolver_result_user_data); + class ChannelConfigHelper { + public: + struct ApplyServiceConfigResult { + // Set to true if the service config has changed since the last result. + bool service_config_changed = false; + // Set to true if we don't have a valid service config to use. + // This tells the ResolvingLoadBalancingPolicy to put the channel + // into TRANSIENT_FAILURE. + bool no_valid_service_config = false; + // A service config parsing error occurred. + grpc_error* service_config_error = GRPC_ERROR_NONE; + // The LB policy config to use. + RefCountedPtr lb_policy_config; + }; + + // Applies the service config to the channel. + virtual ApplyServiceConfigResult ApplyServiceConfig( + const Resolver::Result& result) = 0; + + // Applies the ConfigSelector to the channel. + virtual void ApplyConfigSelector( + bool service_config_changed, + RefCountedPtr config_selector) = 0; + + // Indicates a resolver transient failure. + virtual void ResolverTransientFailure(grpc_error* error) = 0; + }; + + ResolvingLoadBalancingPolicy(Args args, TraceFlag* tracer, + grpc_core::UniquePtr target_uri, + ChannelConfigHelper* helper); virtual const char* name() const override { return "resolving_lb"; } @@ -105,15 +121,16 @@ class ResolvingLoadBalancingPolicy : public LoadBalancingPolicy { // Passed in from caller at construction time. TraceFlag* tracer_; grpc_core::UniquePtr target_uri_; - ProcessResolverResultCallback process_resolver_result_ = nullptr; - void* process_resolver_result_user_data_ = nullptr; - grpc_core::UniquePtr child_policy_name_; - RefCountedPtr child_lb_config_; + ChannelConfigHelper* helper_; // Resolver and associated state. OrphanablePtr resolver_; bool previous_resolution_contained_addresses_ = false; + // Determined by resolver results. + grpc_core::UniquePtr child_policy_name_; + RefCountedPtr child_lb_config_; + // Child LB policy. OrphanablePtr lb_policy_; }; diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 37bed16955f..a25f10239f6 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -23,6 +23,7 @@ CORE_SOURCE_FILES = [ 'src/core/ext/filters/client_channel/client_channel_channelz.cc', 'src/core/ext/filters/client_channel/client_channel_factory.cc', 'src/core/ext/filters/client_channel/client_channel_plugin.cc', + 'src/core/ext/filters/client_channel/config_selector.cc', 'src/core/ext/filters/client_channel/global_subchannel_pool.cc', 'src/core/ext/filters/client_channel/health/health_check_client.cc', 'src/core/ext/filters/client_channel/http_connect_handshaker.cc', diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index e69d345dfbb..1a81817d3a9 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -1081,6 +1081,8 @@ src/core/ext/filters/client_channel/client_channel_channelz.h \ src/core/ext/filters/client_channel/client_channel_factory.cc \ src/core/ext/filters/client_channel/client_channel_factory.h \ src/core/ext/filters/client_channel/client_channel_plugin.cc \ +src/core/ext/filters/client_channel/config_selector.cc \ +src/core/ext/filters/client_channel/config_selector.h \ src/core/ext/filters/client_channel/connector.h \ src/core/ext/filters/client_channel/global_subchannel_pool.cc \ src/core/ext/filters/client_channel/global_subchannel_pool.h \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index bff4f23c4a4..a4443ebd080 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -878,6 +878,8 @@ src/core/ext/filters/client_channel/client_channel_channelz.h \ src/core/ext/filters/client_channel/client_channel_factory.cc \ src/core/ext/filters/client_channel/client_channel_factory.h \ src/core/ext/filters/client_channel/client_channel_plugin.cc \ +src/core/ext/filters/client_channel/config_selector.cc \ +src/core/ext/filters/client_channel/config_selector.h \ src/core/ext/filters/client_channel/connector.h \ src/core/ext/filters/client_channel/global_subchannel_pool.cc \ src/core/ext/filters/client_channel/global_subchannel_pool.h \ From 4009dc1115d2233136e890abf467e4e09171bfb0 Mon Sep 17 00:00:00 2001 From: Chengyuan Zhang Date: Fri, 19 Jun 2020 10:12:49 -0700 Subject: [PATCH 74/80] Add grpc-java v1.30.1 to interop test client matrix --- tools/interop_matrix/client_matrix.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/interop_matrix/client_matrix.py b/tools/interop_matrix/client_matrix.py index 7278730cef6..e38c1c77ee9 100644 --- a/tools/interop_matrix/client_matrix.py +++ b/tools/interop_matrix/client_matrix.py @@ -220,6 +220,7 @@ LANG_RELEASE_MATRIX = { ('v1.28.1', ReleaseInfo()), ('v1.29.0', ReleaseInfo()), ('v1.30.0', ReleaseInfo()), + ('v1.30.1', ReleaseInfo()), ]), 'python': OrderedDict([ From 492889f1c032156a4afcad0cead314a1b8bbbbe0 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Fri, 19 Jun 2020 11:32:18 -0700 Subject: [PATCH 75/80] Convert chttp2 server connection-handling code to C++. --- CMakeLists.txt | 69 +- Makefile | 84 +- build_autogenerated.yaml | 25 +- .../transport/chttp2/server/chttp2_server.cc | 724 +++++++++--------- .../transport/chttp2/server/chttp2_server.h | 8 +- .../chttp2/server/insecure/server_chttp2.cc | 2 +- .../server/secure/server_secure_chttp2.cc | 2 +- src/core/lib/surface/server.cc | 149 ++-- src/core/lib/surface/server.h | 35 +- test/core/surface/BUILD | 3 + test/core/surface/server_chttp2_test.cc | 30 +- tools/run_tests/generated/tests.json | 48 +- 12 files changed, 618 insertions(+), 561 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4e0c695a5bc..97db934faac 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -569,7 +569,6 @@ if(gRPC_BUILD_TESTS) add_dependencies(buildtests_c secure_endpoint_test) add_dependencies(buildtests_c security_connector_test) add_dependencies(buildtests_c sequential_connectivity_test) - add_dependencies(buildtests_c server_chttp2_test) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_c server_ssl_test) endif() @@ -793,6 +792,7 @@ if(gRPC_BUILD_TESTS) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx server_builder_with_socket_mutator_test) endif() + add_dependencies(buildtests_cxx server_chttp2_test) add_dependencies(buildtests_cxx server_context_test_spouse_test) add_dependencies(buildtests_cxx server_early_return_test) add_dependencies(buildtests_cxx server_interceptors_end2end_test) @@ -6830,35 +6830,6 @@ target_link_libraries(sequential_connectivity_test ) -endif() -if(gRPC_BUILD_TESTS) - -add_executable(server_chttp2_test - test/core/surface/server_chttp2_test.cc -) - -target_include_directories(server_chttp2_test - PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}/include - ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} - ${_gRPC_SSL_INCLUDE_DIR} - ${_gRPC_UPB_GENERATED_DIR} - ${_gRPC_UPB_GRPC_GENERATED_DIR} - ${_gRPC_UPB_INCLUDE_DIR} - ${_gRPC_ZLIB_INCLUDE_DIR} -) - -target_link_libraries(server_chttp2_test - ${_gRPC_ALLTARGETS_LIBRARIES} - grpc_test_util - grpc - gpr - address_sorting - upb -) - - endif() if(gRPC_BUILD_TESTS) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) @@ -13004,6 +12975,44 @@ endif() endif() if(gRPC_BUILD_TESTS) +add_executable(server_chttp2_test + test/core/surface/server_chttp2_test.cc + third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc +) + +target_include_directories(server_chttp2_test + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} + third_party/googletest/googletest/include + third_party/googletest/googletest + third_party/googletest/googlemock/include + third_party/googletest/googlemock + ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(server_chttp2_test + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_test_util + grpc + gpr + address_sorting + upb + ${_gRPC_GFLAGS_LIBRARIES} +) + + +endif() +if(gRPC_BUILD_TESTS) + add_executable(server_context_test_spouse_test test/cpp/test/server_context_test_spouse_test.cc third_party/googletest/googletest/src/gtest-all.cc diff --git a/Makefile b/Makefile index 83c7600a698..bc95743cad9 100644 --- a/Makefile +++ b/Makefile @@ -1112,7 +1112,6 @@ secure_channel_create_test: $(BINDIR)/$(CONFIG)/secure_channel_create_test secure_endpoint_test: $(BINDIR)/$(CONFIG)/secure_endpoint_test security_connector_test: $(BINDIR)/$(CONFIG)/security_connector_test sequential_connectivity_test: $(BINDIR)/$(CONFIG)/sequential_connectivity_test -server_chttp2_test: $(BINDIR)/$(CONFIG)/server_chttp2_test server_ssl_test: $(BINDIR)/$(CONFIG)/server_ssl_test server_test: $(BINDIR)/$(CONFIG)/server_test slice_buffer_test: $(BINDIR)/$(CONFIG)/slice_buffer_test @@ -1269,6 +1268,7 @@ secure_auth_context_test: $(BINDIR)/$(CONFIG)/secure_auth_context_test server_builder_plugin_test: $(BINDIR)/$(CONFIG)/server_builder_plugin_test server_builder_test: $(BINDIR)/$(CONFIG)/server_builder_test server_builder_with_socket_mutator_test: $(BINDIR)/$(CONFIG)/server_builder_with_socket_mutator_test +server_chttp2_test: $(BINDIR)/$(CONFIG)/server_chttp2_test server_context_test_spouse_test: $(BINDIR)/$(CONFIG)/server_context_test_spouse_test server_early_return_test: $(BINDIR)/$(CONFIG)/server_early_return_test server_fuzzer: $(BINDIR)/$(CONFIG)/server_fuzzer @@ -1487,7 +1487,6 @@ buildtests_c: privatelibs_c \ $(BINDIR)/$(CONFIG)/secure_endpoint_test \ $(BINDIR)/$(CONFIG)/security_connector_test \ $(BINDIR)/$(CONFIG)/sequential_connectivity_test \ - $(BINDIR)/$(CONFIG)/server_chttp2_test \ $(BINDIR)/$(CONFIG)/server_ssl_test \ $(BINDIR)/$(CONFIG)/server_test \ $(BINDIR)/$(CONFIG)/slice_buffer_test \ @@ -1631,6 +1630,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/server_builder_plugin_test \ $(BINDIR)/$(CONFIG)/server_builder_test \ $(BINDIR)/$(CONFIG)/server_builder_with_socket_mutator_test \ + $(BINDIR)/$(CONFIG)/server_chttp2_test \ $(BINDIR)/$(CONFIG)/server_context_test_spouse_test \ $(BINDIR)/$(CONFIG)/server_early_return_test \ $(BINDIR)/$(CONFIG)/server_interceptors_end2end_test \ @@ -1788,6 +1788,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/server_builder_plugin_test \ $(BINDIR)/$(CONFIG)/server_builder_test \ $(BINDIR)/$(CONFIG)/server_builder_with_socket_mutator_test \ + $(BINDIR)/$(CONFIG)/server_chttp2_test \ $(BINDIR)/$(CONFIG)/server_context_test_spouse_test \ $(BINDIR)/$(CONFIG)/server_early_return_test \ $(BINDIR)/$(CONFIG)/server_interceptors_end2end_test \ @@ -2043,8 +2044,6 @@ test_c: buildtests_c $(Q) $(BINDIR)/$(CONFIG)/secure_endpoint_test || ( echo test secure_endpoint_test failed ; exit 1 ) $(E) "[RUN] Testing security_connector_test" $(Q) $(BINDIR)/$(CONFIG)/security_connector_test || ( echo test security_connector_test failed ; exit 1 ) - $(E) "[RUN] Testing server_chttp2_test" - $(Q) $(BINDIR)/$(CONFIG)/server_chttp2_test || ( echo test server_chttp2_test failed ; exit 1 ) $(E) "[RUN] Testing server_ssl_test" $(Q) $(BINDIR)/$(CONFIG)/server_ssl_test || ( echo test server_ssl_test failed ; exit 1 ) $(E) "[RUN] Testing server_test" @@ -2299,6 +2298,8 @@ test_cxx: buildtests_cxx $(Q) $(BINDIR)/$(CONFIG)/server_builder_test || ( echo test server_builder_test failed ; exit 1 ) $(E) "[RUN] Testing server_builder_with_socket_mutator_test" $(Q) $(BINDIR)/$(CONFIG)/server_builder_with_socket_mutator_test || ( echo test server_builder_with_socket_mutator_test failed ; exit 1 ) + $(E) "[RUN] Testing server_chttp2_test" + $(Q) $(BINDIR)/$(CONFIG)/server_chttp2_test || ( echo test server_chttp2_test failed ; exit 1 ) $(E) "[RUN] Testing server_context_test_spouse_test" $(Q) $(BINDIR)/$(CONFIG)/server_context_test_spouse_test || ( echo test server_context_test_spouse_test failed ; exit 1 ) $(E) "[RUN] Testing server_early_return_test" @@ -10056,38 +10057,6 @@ endif endif -SERVER_CHTTP2_TEST_SRC = \ - test/core/surface/server_chttp2_test.cc \ - -SERVER_CHTTP2_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(SERVER_CHTTP2_TEST_SRC)))) -ifeq ($(NO_SECURE),true) - -# You can't build secure targets if you don't have OpenSSL. - -$(BINDIR)/$(CONFIG)/server_chttp2_test: openssl_dep_error - -else - - - -$(BINDIR)/$(CONFIG)/server_chttp2_test: $(SERVER_CHTTP2_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a - $(E) "[LD] Linking $@" - $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) $(SERVER_CHTTP2_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/server_chttp2_test - -endif - -$(OBJDIR)/$(CONFIG)/test/core/surface/server_chttp2_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a - -deps_server_chttp2_test: $(SERVER_CHTTP2_TEST_OBJS:.o=.dep) - -ifneq ($(NO_SECURE),true) -ifneq ($(NO_DEPS),true) --include $(SERVER_CHTTP2_TEST_OBJS:.o=.dep) -endif -endif - - SERVER_SSL_TEST_SRC = \ test/core/handshake/server_ssl.cc \ test/core/handshake/server_ssl_common.cc \ @@ -17223,6 +17192,49 @@ endif $(OBJDIR)/$(CONFIG)/test/cpp/server/server_builder_with_socket_mutator_test.o: $(GENDIR)/src/proto/grpc/testing/echo.pb.cc $(GENDIR)/src/proto/grpc/testing/echo.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/echo_messages.pb.cc $(GENDIR)/src/proto/grpc/testing/echo_messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/simple_messages.pb.cc $(GENDIR)/src/proto/grpc/testing/simple_messages.grpc.pb.cc +SERVER_CHTTP2_TEST_SRC = \ + test/core/surface/server_chttp2_test.cc \ + +SERVER_CHTTP2_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(SERVER_CHTTP2_TEST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/server_chttp2_test: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.5.0+. + +$(BINDIR)/$(CONFIG)/server_chttp2_test: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/server_chttp2_test: $(PROTOBUF_DEP) $(SERVER_CHTTP2_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(SERVER_CHTTP2_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/server_chttp2_test + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/core/surface/server_chttp2_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libupb.a + +deps_server_chttp2_test: $(SERVER_CHTTP2_TEST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(SERVER_CHTTP2_TEST_OBJS:.o=.dep) +endif +endif + + SERVER_CONTEXT_TEST_SPOUSE_TEST_SRC = \ test/cpp/test/server_context_test_spouse_test.cc \ diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index 5f09c7b3e73..294f1609ece 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -4136,18 +4136,6 @@ targets: - gpr - address_sorting - upb -- name: server_chttp2_test - build: test - language: c - headers: [] - src: - - test/core/surface/server_chttp2_test.cc - deps: - - grpc_test_util - - grpc - - gpr - - address_sorting - - upb - name: server_ssl_test build: test language: c @@ -6932,6 +6920,19 @@ targets: - linux - posix - mac +- name: server_chttp2_test + gtest: true + build: test + language: c++ + headers: [] + src: + - test/core/surface/server_chttp2_test.cc + deps: + - grpc_test_util + - grpc + - gpr + - address_sorting + - upb - name: server_context_test_spouse_test gtest: true build: test diff --git a/src/core/ext/transport/chttp2/server/chttp2_server.cc b/src/core/ext/transport/chttp2/server/chttp2_server.cc index 446a214c183..b173ba02499 100644 --- a/src/core/ext/transport/chttp2/server/chttp2_server.cc +++ b/src/core/ext/transport/chttp2/server/chttp2_server.cc @@ -39,6 +39,8 @@ #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/channel/handshaker.h" #include "src/core/lib/channel/handshaker_registry.h" +#include "src/core/lib/gprpp/ref_counted.h" +#include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/iomgr/endpoint.h" #include "src/core/lib/iomgr/resolve_address.h" #include "src/core/lib/iomgr/resource_quota.h" @@ -47,405 +49,439 @@ #include "src/core/lib/surface/api_trace.h" #include "src/core/lib/surface/server.h" -struct server_state { - grpc_server* server; - grpc_tcp_server* tcp_server; - grpc_channel_args* args; - gpr_mu mu; - bool shutdown; - grpc_closure tcp_server_shutdown_complete; - grpc_closure* server_destroy_listener_done; - grpc_core::HandshakeManager* pending_handshake_mgrs; - grpc_core::RefCountedPtr - channelz_listen_socket; -}; +namespace grpc_core { +namespace { + +class Chttp2ServerListener : public ServerListenerInterface { + public: + static grpc_error* Create(grpc_server* server, const char* addr, + grpc_channel_args* args, int* port_num); + + static grpc_error* CreateWithAcceptor(grpc_server* server, const char* name, + grpc_channel_args* args); + + // Do not instantiate directly. Use one of the factory methods above. + Chttp2ServerListener(grpc_server* server, grpc_channel_args* args); + ~Chttp2ServerListener(); + + void Start(grpc_server* server, grpc_pollset** pollsets, + size_t npollsets) override; + + channelz::ListenSocketNode* channelz_listen_socket_node() const override { + return channelz_listen_socket_.get(); + } + + void SetOnDestroyDone(grpc_closure* on_destroy_done) override; + + void Orphan() override; + + private: + class ConnectionState : public RefCounted { + public: + ConnectionState(Chttp2ServerListener* listener, + grpc_pollset* accepting_pollset, + grpc_tcp_server_acceptor* acceptor, + RefCountedPtr handshake_mgr, + grpc_channel_args* args, grpc_endpoint* endpoint); + + ~ConnectionState(); + + private: + static void OnTimeout(void* arg, grpc_error* error); + static void OnReceiveSettings(void* arg, grpc_error* error); + static void OnHandshakeDone(void* arg, grpc_error* error); -struct server_connection_state { - gpr_refcount refs; - server_state* svr_state; - grpc_pollset* accepting_pollset; - grpc_tcp_server_acceptor* acceptor; - grpc_core::RefCountedPtr handshake_mgr; - // State for enforcing handshake timeout on receiving HTTP/2 settings. - grpc_chttp2_transport* transport; - grpc_millis deadline; - grpc_timer timer; - grpc_closure on_timeout; - grpc_closure on_receive_settings; - grpc_pollset_set* interested_parties; + Chttp2ServerListener* const listener_; + grpc_pollset* const accepting_pollset_; + grpc_tcp_server_acceptor* const acceptor_; + RefCountedPtr handshake_mgr_; + // State for enforcing handshake timeout on receiving HTTP/2 settings. + grpc_chttp2_transport* transport_ = nullptr; + grpc_millis deadline_; + grpc_timer timer_; + grpc_closure on_timeout_; + grpc_closure on_receive_settings_; + grpc_pollset_set* const interested_parties_; + }; + + static void OnAccept(void* arg, grpc_endpoint* tcp, + grpc_pollset* accepting_pollset, + grpc_tcp_server_acceptor* acceptor); + + RefCountedPtr CreateHandshakeManager(); + + static void TcpServerShutdownComplete(void* arg, grpc_error* error); + + static void DestroyListener(grpc_server* /*server*/, void* arg, + grpc_closure* destroy_done); + + grpc_server* const server_; + grpc_channel_args* const args_; + grpc_tcp_server* tcp_server_; + Mutex mu_; + bool shutdown_ = true; + grpc_closure tcp_server_shutdown_complete_; + grpc_closure* on_destroy_done_ = nullptr; + HandshakeManager* pending_handshake_mgrs_ = nullptr; + RefCountedPtr channelz_listen_socket_; }; -static void server_connection_state_unref( - server_connection_state* connection_state) { - if (gpr_unref(&connection_state->refs)) { - if (connection_state->transport != nullptr) { - GRPC_CHTTP2_UNREF_TRANSPORT(connection_state->transport, - "receive settings timeout"); - } - grpc_pollset_set_del_pollset(connection_state->interested_parties, - connection_state->accepting_pollset); - grpc_pollset_set_destroy(connection_state->interested_parties); - gpr_free(connection_state); +// +// Chttp2ServerListener::ConnectionState +// + +grpc_millis GetConnectionDeadline(const grpc_channel_args* args) { + int timeout_ms = + grpc_channel_args_find_integer(args, GRPC_ARG_SERVER_HANDSHAKE_TIMEOUT_MS, + {120 * GPR_MS_PER_SEC, 1, INT_MAX}); + return ExecCtx::Get()->Now() + timeout_ms; +} + +Chttp2ServerListener::ConnectionState::ConnectionState( + Chttp2ServerListener* listener, grpc_pollset* accepting_pollset, + grpc_tcp_server_acceptor* acceptor, + RefCountedPtr handshake_mgr, grpc_channel_args* args, + grpc_endpoint* endpoint) + : listener_(listener), + accepting_pollset_(accepting_pollset), + acceptor_(acceptor), + handshake_mgr_(std::move(handshake_mgr)), + deadline_(GetConnectionDeadline(args)), + interested_parties_(grpc_pollset_set_create()) { + grpc_pollset_set_add_pollset(interested_parties_, accepting_pollset_); + HandshakerRegistry::AddHandshakers(HANDSHAKER_SERVER, args, + interested_parties_, handshake_mgr_.get()); + handshake_mgr_->DoHandshake(endpoint, args, deadline_, acceptor_, + OnHandshakeDone, this); +} + +Chttp2ServerListener::ConnectionState::~ConnectionState() { + if (transport_ != nullptr) { + GRPC_CHTTP2_UNREF_TRANSPORT(transport_, "receive settings timeout"); } + grpc_pollset_set_del_pollset(interested_parties_, accepting_pollset_); + grpc_pollset_set_destroy(interested_parties_); } -static void on_timeout(void* arg, grpc_error* error) { - server_connection_state* connection_state = - static_cast(arg); +void Chttp2ServerListener::ConnectionState::OnTimeout(void* arg, + grpc_error* error) { + ConnectionState* self = static_cast(arg); // Note that we may be called with GRPC_ERROR_NONE when the timer fires // or with an error indicating that the timer system is being shut down. if (error != GRPC_ERROR_CANCELLED) { grpc_transport_op* op = grpc_make_transport_op(nullptr); op->disconnect_with_error = GRPC_ERROR_CREATE_FROM_STATIC_STRING( "Did not receive HTTP/2 settings before handshake timeout"); - grpc_transport_perform_op(&connection_state->transport->base, op); + grpc_transport_perform_op(&self->transport_->base, op); } - server_connection_state_unref(connection_state); + self->Unref(); } -static void on_receive_settings(void* arg, grpc_error* error) { - server_connection_state* connection_state = - static_cast(arg); +void Chttp2ServerListener::ConnectionState::OnReceiveSettings( + void* arg, grpc_error* error) { + ConnectionState* self = static_cast(arg); if (error == GRPC_ERROR_NONE) { - grpc_timer_cancel(&connection_state->timer); + grpc_timer_cancel(&self->timer_); } - server_connection_state_unref(connection_state); + self->Unref(); } -static void on_handshake_done(void* arg, grpc_error* error) { - auto* args = static_cast(arg); - server_connection_state* connection_state = - static_cast(args->user_data); - gpr_mu_lock(&connection_state->svr_state->mu); - grpc_resource_user* resource_user = grpc_server_get_default_resource_user( - connection_state->svr_state->server); - if (error != GRPC_ERROR_NONE || connection_state->svr_state->shutdown) { - const char* error_str = grpc_error_string(error); - gpr_log(GPR_DEBUG, "Handshaking failed: %s", error_str); - grpc_resource_user* resource_user = grpc_server_get_default_resource_user( - connection_state->svr_state->server); - if (resource_user != nullptr) { - grpc_resource_user_free(resource_user, GRPC_RESOURCE_QUOTA_CHANNEL_SIZE); - } - if (error == GRPC_ERROR_NONE && args->endpoint != nullptr) { - // We were shut down after handshaking completed successfully, so - // destroy the endpoint here. - // TODO(ctiller): It is currently necessary to shutdown endpoints - // before destroying them, even if we know that there are no - // pending read/write callbacks. This should be fixed, at which - // point this can be removed. - grpc_endpoint_shutdown(args->endpoint, GRPC_ERROR_NONE); - grpc_endpoint_destroy(args->endpoint); - grpc_channel_args_destroy(args->args); - grpc_slice_buffer_destroy_internal(args->read_buffer); - gpr_free(args->read_buffer); - } - } else { - // If the handshaking succeeded but there is no endpoint, then the - // handshaker may have handed off the connection to some external - // code, so we can just clean up here without creating a transport. - if (args->endpoint != nullptr) { - grpc_transport* transport = grpc_create_chttp2_transport( - args->args, args->endpoint, false, resource_user); - grpc_server_setup_transport( - connection_state->svr_state->server, transport, - connection_state->accepting_pollset, args->args, - grpc_chttp2_transport_get_socket_node(transport), resource_user); - // Use notify_on_receive_settings callback to enforce the - // handshake deadline. - connection_state->transport = - reinterpret_cast(transport); - gpr_ref(&connection_state->refs); - GRPC_CLOSURE_INIT(&connection_state->on_receive_settings, - on_receive_settings, connection_state, - grpc_schedule_on_exec_ctx); - grpc_chttp2_transport_start_reading( - transport, args->read_buffer, &connection_state->on_receive_settings); - grpc_channel_args_destroy(args->args); - gpr_ref(&connection_state->refs); - GRPC_CHTTP2_REF_TRANSPORT((grpc_chttp2_transport*)transport, - "receive settings timeout"); - GRPC_CLOSURE_INIT(&connection_state->on_timeout, on_timeout, - connection_state, grpc_schedule_on_exec_ctx); - grpc_timer_init(&connection_state->timer, connection_state->deadline, - &connection_state->on_timeout); - } else { +void Chttp2ServerListener::ConnectionState::OnHandshakeDone(void* arg, + grpc_error* error) { + auto* args = static_cast(arg); + ConnectionState* self = static_cast(args->user_data); + { + MutexLock lock(&self->listener_->mu_); + grpc_resource_user* resource_user = + grpc_server_get_default_resource_user(self->listener_->server_); + if (error != GRPC_ERROR_NONE || self->listener_->shutdown_) { + const char* error_str = grpc_error_string(error); + gpr_log(GPR_DEBUG, "Handshaking failed: %s", error_str); + grpc_resource_user* resource_user = + grpc_server_get_default_resource_user(self->listener_->server_); if (resource_user != nullptr) { grpc_resource_user_free(resource_user, GRPC_RESOURCE_QUOTA_CHANNEL_SIZE); } + if (error == GRPC_ERROR_NONE && args->endpoint != nullptr) { + // We were shut down after handshaking completed successfully, so + // destroy the endpoint here. + // TODO(ctiller): It is currently necessary to shutdown endpoints + // before destroying them, even if we know that there are no + // pending read/write callbacks. This should be fixed, at which + // point this can be removed. + grpc_endpoint_shutdown(args->endpoint, GRPC_ERROR_NONE); + grpc_endpoint_destroy(args->endpoint); + grpc_channel_args_destroy(args->args); + grpc_slice_buffer_destroy_internal(args->read_buffer); + gpr_free(args->read_buffer); + } + } else { + // If the handshaking succeeded but there is no endpoint, then the + // handshaker may have handed off the connection to some external + // code, so we can just clean up here without creating a transport. + if (args->endpoint != nullptr) { + grpc_transport* transport = grpc_create_chttp2_transport( + args->args, args->endpoint, false, resource_user); + grpc_server_setup_transport( + self->listener_->server_, transport, self->accepting_pollset_, + args->args, grpc_chttp2_transport_get_socket_node(transport), + resource_user); + // Use notify_on_receive_settings callback to enforce the + // handshake deadline. + // Note: The reinterpret_cast<>s here are safe, because + // grpc_chttp2_transport is a C-style extension of + // grpc_transport, so this is morally equivalent of a + // static_cast<> to a derived class. + // TODO(roth): Change to static_cast<> when we C++-ify the + // transport API. + self->transport_ = reinterpret_cast(transport); + self->Ref().release(); // Held by OnReceiveSettings(). + GRPC_CLOSURE_INIT(&self->on_receive_settings_, OnReceiveSettings, self, + grpc_schedule_on_exec_ctx); + grpc_chttp2_transport_start_reading(transport, args->read_buffer, + &self->on_receive_settings_); + grpc_channel_args_destroy(args->args); + self->Ref().release(); // Held by OnTimeout(). + GRPC_CHTTP2_REF_TRANSPORT( + reinterpret_cast(transport), + "receive settings timeout"); + GRPC_CLOSURE_INIT(&self->on_timeout_, OnTimeout, self, + grpc_schedule_on_exec_ctx); + grpc_timer_init(&self->timer_, self->deadline_, &self->on_timeout_); + } else { + if (resource_user != nullptr) { + grpc_resource_user_free(resource_user, + GRPC_RESOURCE_QUOTA_CHANNEL_SIZE); + } + } } + self->handshake_mgr_->RemoveFromPendingMgrList( + &self->listener_->pending_handshake_mgrs_); } - connection_state->handshake_mgr->RemoveFromPendingMgrList( - &connection_state->svr_state->pending_handshake_mgrs); - gpr_mu_unlock(&connection_state->svr_state->mu); - connection_state->handshake_mgr.reset(); - gpr_free(connection_state->acceptor); - grpc_tcp_server_unref(connection_state->svr_state->tcp_server); - server_connection_state_unref(connection_state); + self->handshake_mgr_.reset(); + gpr_free(self->acceptor_); + grpc_tcp_server_unref(self->listener_->tcp_server_); + self->Unref(); } -static void on_accept(void* arg, grpc_endpoint* tcp, - grpc_pollset* accepting_pollset, - grpc_tcp_server_acceptor* acceptor) { - server_state* state = static_cast(arg); - gpr_mu_lock(&state->mu); - if (state->shutdown) { - gpr_mu_unlock(&state->mu); - grpc_endpoint_shutdown(tcp, GRPC_ERROR_NONE); - grpc_endpoint_destroy(tcp); - gpr_free(acceptor); - return; +// +// Chttp2ServerListener +// + +grpc_error* Chttp2ServerListener::Create(grpc_server* server, const char* addr, + grpc_channel_args* args, + int* port_num) { + std::vector error_list; + grpc_resolved_addresses* resolved = nullptr; + Chttp2ServerListener* listener = nullptr; + // The bulk of this method is inside of a lambda to make cleanup + // easier without using goto. + grpc_error* error = [&]() { + *port_num = -1; + /* resolve address */ + grpc_error* error = grpc_blocking_resolve_address(addr, "https", &resolved); + if (error != GRPC_ERROR_NONE) return error; + // Create Chttp2ServerListener. + listener = new Chttp2ServerListener(server, args); + error = grpc_tcp_server_create(&listener->tcp_server_shutdown_complete_, + args, &listener->tcp_server_); + if (error != GRPC_ERROR_NONE) return error; + for (size_t i = 0; i < resolved->naddrs; i++) { + int port_temp; + error = grpc_tcp_server_add_port(listener->tcp_server_, + &resolved->addrs[i], &port_temp); + if (error != GRPC_ERROR_NONE) { + error_list.push_back(error); + } else { + if (*port_num == -1) { + *port_num = port_temp; + } else { + GPR_ASSERT(*port_num == port_temp); + } + } + } + if (error_list.size() == resolved->naddrs) { + std::string msg = + absl::StrFormat("No address added out of total %" PRIuPTR " resolved", + resolved->naddrs); + return GRPC_ERROR_CREATE_REFERENCING_FROM_COPIED_STRING( + msg.c_str(), error_list.data(), error_list.size()); + } else if (!error_list.empty()) { + std::string msg = absl::StrFormat( + "Only %" PRIuPTR " addresses added out of total %" PRIuPTR + " resolved", + resolved->naddrs - error_list.size(), resolved->naddrs); + error = GRPC_ERROR_CREATE_REFERENCING_FROM_COPIED_STRING( + msg.c_str(), error_list.data(), error_list.size()); + gpr_log(GPR_INFO, "WARNING: %s", grpc_error_string(error)); + GRPC_ERROR_UNREF(error); + /* we managed to bind some addresses: continue */ + } + // Create channelz node. + if (grpc_channel_args_find_bool(args, GRPC_ARG_ENABLE_CHANNELZ, + GRPC_ENABLE_CHANNELZ_DEFAULT)) { + listener->channelz_listen_socket_ = + MakeRefCounted( + addr, absl::StrFormat("chttp2 listener %s", addr)); + } + /* Register with the server only upon success */ + grpc_server_add_listener(server, + OrphanablePtr(listener)); + return GRPC_ERROR_NONE; + }(); + if (resolved != nullptr) { + grpc_resolved_addresses_destroy(resolved); + } + if (error != GRPC_ERROR_NONE) { + if (listener != nullptr) { + if (listener->tcp_server_ != nullptr) { + grpc_tcp_server_unref(listener->tcp_server_); + } else { + delete listener; + } + } else { + grpc_channel_args_destroy(args); + } + *port_num = 0; + } + for (grpc_error* error : error_list) { + GRPC_ERROR_UNREF(error); } + return error; +} + +grpc_error* Chttp2ServerListener::CreateWithAcceptor(grpc_server* server, + const char* name, + grpc_channel_args* args) { + Chttp2ServerListener* listener = new Chttp2ServerListener(server, args); + grpc_error* error = grpc_tcp_server_create( + &listener->tcp_server_shutdown_complete_, args, &listener->tcp_server_); + if (error != GRPC_ERROR_NONE) { + delete listener; + return error; + } + // TODO(yangg) channelz + TcpServerFdHandler** arg_val = + grpc_channel_args_find_pointer(args, name); + *arg_val = grpc_tcp_server_create_fd_handler(listener->tcp_server_); + grpc_server_add_listener(server, + OrphanablePtr(listener)); + return GRPC_ERROR_NONE; +} + +Chttp2ServerListener::Chttp2ServerListener(grpc_server* server, + grpc_channel_args* args) + : server_(server), args_(args) { + GRPC_CLOSURE_INIT(&tcp_server_shutdown_complete_, TcpServerShutdownComplete, + this, grpc_schedule_on_exec_ctx); +} + +Chttp2ServerListener::~Chttp2ServerListener() { + grpc_channel_args_destroy(args_); +} + +/* Server callback: start listening on our ports */ +void Chttp2ServerListener::Start(grpc_server* /*server*/, + grpc_pollset** pollsets, + size_t pollset_count) { + { + MutexLock lock(&mu_); + shutdown_ = false; + } + grpc_tcp_server_start(tcp_server_, pollsets, pollset_count, OnAccept, this); +} + +void Chttp2ServerListener::SetOnDestroyDone(grpc_closure* on_destroy_done) { + MutexLock lock(&mu_); + on_destroy_done_ = on_destroy_done; +} + +RefCountedPtr Chttp2ServerListener::CreateHandshakeManager() { + MutexLock lock(&mu_); + if (shutdown_) return nullptr; grpc_resource_user* resource_user = - grpc_server_get_default_resource_user(state->server); + grpc_server_get_default_resource_user(server_); if (resource_user != nullptr && !grpc_resource_user_safe_alloc(resource_user, GRPC_RESOURCE_QUOTA_CHANNEL_SIZE)) { - gpr_log( - GPR_ERROR, - "Memory quota exhausted, rejecting the connection, no handshaking."); - gpr_mu_unlock(&state->mu); + gpr_log(GPR_ERROR, + "Memory quota exhausted, rejecting connection, no handshaking."); + return nullptr; + } + auto handshake_mgr = MakeRefCounted(); + handshake_mgr->AddToPendingMgrList(&pending_handshake_mgrs_); + grpc_tcp_server_ref(tcp_server_); // Ref held by ConnectionState. + return handshake_mgr; +} + +void Chttp2ServerListener::OnAccept(void* arg, grpc_endpoint* tcp, + grpc_pollset* accepting_pollset, + grpc_tcp_server_acceptor* acceptor) { + Chttp2ServerListener* self = static_cast(arg); + RefCountedPtr handshake_mgr = + self->CreateHandshakeManager(); + if (handshake_mgr == nullptr) { grpc_endpoint_shutdown(tcp, GRPC_ERROR_NONE); grpc_endpoint_destroy(tcp); gpr_free(acceptor); return; } - auto handshake_mgr = grpc_core::MakeRefCounted(); - handshake_mgr->AddToPendingMgrList(&state->pending_handshake_mgrs); - grpc_tcp_server_ref(state->tcp_server); - gpr_mu_unlock(&state->mu); - server_connection_state* connection_state = - static_cast( - gpr_zalloc(sizeof(*connection_state))); - gpr_ref_init(&connection_state->refs, 1); - connection_state->svr_state = state; - connection_state->accepting_pollset = accepting_pollset; - connection_state->acceptor = acceptor; - connection_state->handshake_mgr = handshake_mgr; - connection_state->interested_parties = grpc_pollset_set_create(); - grpc_pollset_set_add_pollset(connection_state->interested_parties, - connection_state->accepting_pollset); - grpc_core::HandshakerRegistry::AddHandshakers( - grpc_core::HANDSHAKER_SERVER, state->args, - connection_state->interested_parties, - connection_state->handshake_mgr.get()); - const grpc_arg* timeout_arg = - grpc_channel_args_find(state->args, GRPC_ARG_SERVER_HANDSHAKE_TIMEOUT_MS); - connection_state->deadline = - grpc_core::ExecCtx::Get()->Now() + - grpc_channel_arg_get_integer(timeout_arg, - {120 * GPR_MS_PER_SEC, 1, INT_MAX}); - connection_state->handshake_mgr->DoHandshake( - tcp, state->args, connection_state->deadline, acceptor, on_handshake_done, - connection_state); -} - -/* Server callback: start listening on our ports */ -static void server_start_listener(grpc_server* /*server*/, void* arg, - grpc_pollset** pollsets, - size_t pollset_count) { - server_state* state = static_cast(arg); - gpr_mu_lock(&state->mu); - state->shutdown = false; - gpr_mu_unlock(&state->mu); - grpc_tcp_server_start(state->tcp_server, pollsets, pollset_count, on_accept, - state); + // Deletes itself when done. + new ConnectionState(self, accepting_pollset, acceptor, + std::move(handshake_mgr), self->args_, tcp); } -static void tcp_server_shutdown_complete(void* arg, grpc_error* error) { - server_state* state = static_cast(arg); +void Chttp2ServerListener::TcpServerShutdownComplete(void* arg, + grpc_error* error) { + Chttp2ServerListener* self = static_cast(arg); /* ensure all threads have unlocked */ - gpr_mu_lock(&state->mu); - grpc_closure* destroy_done = state->server_destroy_listener_done; - GPR_ASSERT(state->shutdown); - if (state->pending_handshake_mgrs != nullptr) { - state->pending_handshake_mgrs->ShutdownAllPending(GRPC_ERROR_REF(error)); + grpc_closure* destroy_done = nullptr; + { + MutexLock lock(&self->mu_); + destroy_done = self->on_destroy_done_; + GPR_ASSERT(self->shutdown_); + if (self->pending_handshake_mgrs_ != nullptr) { + self->pending_handshake_mgrs_->ShutdownAllPending(GRPC_ERROR_REF(error)); + } + self->channelz_listen_socket_.reset(); } - state->channelz_listen_socket.reset(); - gpr_mu_unlock(&state->mu); // Flush queued work before destroying handshaker factory, since that // may do a synchronous unref. - grpc_core::ExecCtx::Get()->Flush(); + ExecCtx::Get()->Flush(); if (destroy_done != nullptr) { - grpc_core::ExecCtx::Run(DEBUG_LOCATION, destroy_done, - GRPC_ERROR_REF(error)); - grpc_core::ExecCtx::Get()->Flush(); + ExecCtx::Run(DEBUG_LOCATION, destroy_done, GRPC_ERROR_REF(error)); + ExecCtx::Get()->Flush(); } - grpc_channel_args_destroy(state->args); - gpr_mu_destroy(&state->mu); - gpr_free(state); + delete self; } /* Server callback: destroy the tcp listener (so we don't generate further callbacks) */ -static void server_destroy_listener(grpc_server* /*server*/, void* arg, - grpc_closure* destroy_done) { - server_state* state = static_cast(arg); - gpr_mu_lock(&state->mu); - state->shutdown = true; - state->server_destroy_listener_done = destroy_done; - grpc_tcp_server* tcp_server = state->tcp_server; - gpr_mu_unlock(&state->mu); +void Chttp2ServerListener::Orphan() { + grpc_tcp_server* tcp_server; + { + MutexLock lock(&mu_); + shutdown_ = true; + tcp_server = tcp_server_; + } grpc_tcp_server_shutdown_listeners(tcp_server); grpc_tcp_server_unref(tcp_server); } -static grpc_error* chttp2_server_add_acceptor(grpc_server* server, - const char* name, - grpc_channel_args* args) { - grpc_tcp_server* tcp_server = nullptr; - grpc_error* err = GRPC_ERROR_NONE; - server_state* state = nullptr; - const grpc_arg* arg = nullptr; - grpc_core::TcpServerFdHandler** arg_val = nullptr; - state = static_cast(gpr_zalloc(sizeof(*state))); - GRPC_CLOSURE_INIT(&state->tcp_server_shutdown_complete, - tcp_server_shutdown_complete, state, - grpc_schedule_on_exec_ctx); - err = grpc_tcp_server_create(&state->tcp_server_shutdown_complete, args, - &tcp_server); - if (err != GRPC_ERROR_NONE) { - goto error; - } - state->server = server; - state->tcp_server = tcp_server; - state->args = args; - state->shutdown = true; - gpr_mu_init(&state->mu); - // TODO(yangg) channelz - arg = grpc_channel_args_find(args, name); - GPR_ASSERT(arg->type == GRPC_ARG_POINTER); - arg_val = static_cast(arg->value.pointer.p); - *arg_val = grpc_tcp_server_create_fd_handler(tcp_server); - - grpc_server_add_listener(server, state, server_start_listener, - server_destroy_listener, /* node */ nullptr); - return err; - -/* Error path: cleanup and return */ -error: - GPR_ASSERT(err != GRPC_ERROR_NONE); - if (tcp_server) { - grpc_tcp_server_unref(tcp_server); - } else { - grpc_channel_args_destroy(args); - gpr_free(state); - } - return err; -} +} // namespace -grpc_error* grpc_chttp2_server_add_port(grpc_server* server, const char* addr, - grpc_channel_args* args, - int* port_num) { - grpc_resolved_addresses* resolved = nullptr; - grpc_tcp_server* tcp_server = nullptr; - size_t i; - size_t count = 0; - int port_temp; - grpc_error* err = GRPC_ERROR_NONE; - server_state* state = nullptr; - grpc_error** errors = nullptr; - size_t naddrs = 0; - const grpc_arg* arg = nullptr; - - *port_num = -1; +// +// Chttp2ServerAddPort() +// +grpc_error* Chttp2ServerAddPort(grpc_server* server, const char* addr, + grpc_channel_args* args, int* port_num) { if (strncmp(addr, "external:", 9) == 0) { - return chttp2_server_add_acceptor(server, addr, args); - } - - /* resolve address */ - err = grpc_blocking_resolve_address(addr, "https", &resolved); - if (err != GRPC_ERROR_NONE) { - goto error; - } - state = static_cast(gpr_zalloc(sizeof(*state))); - GRPC_CLOSURE_INIT(&state->tcp_server_shutdown_complete, - tcp_server_shutdown_complete, state, - grpc_schedule_on_exec_ctx); - err = grpc_tcp_server_create(&state->tcp_server_shutdown_complete, args, - &tcp_server); - if (err != GRPC_ERROR_NONE) { - goto error; - } - - state->server = server; - state->tcp_server = tcp_server; - state->args = args; - state->shutdown = true; - gpr_mu_init(&state->mu); - - naddrs = resolved->naddrs; - errors = static_cast(gpr_malloc(sizeof(*errors) * naddrs)); - for (i = 0; i < naddrs; i++) { - errors[i] = - grpc_tcp_server_add_port(tcp_server, &resolved->addrs[i], &port_temp); - if (errors[i] == GRPC_ERROR_NONE) { - if (*port_num == -1) { - *port_num = port_temp; - } else { - GPR_ASSERT(*port_num == port_temp); - } - count++; - } + return grpc_core::Chttp2ServerListener::CreateWithAcceptor(server, addr, + args); } - if (count == 0) { - char* msg; - gpr_asprintf(&msg, "No address added out of total %" PRIuPTR " resolved", - naddrs); - err = GRPC_ERROR_CREATE_REFERENCING_FROM_COPIED_STRING(msg, errors, naddrs); - gpr_free(msg); - goto error; - } else if (count != naddrs) { - char* msg; - gpr_asprintf(&msg, - "Only %" PRIuPTR " addresses added out of total %" PRIuPTR - " resolved", - count, naddrs); - err = GRPC_ERROR_CREATE_REFERENCING_FROM_COPIED_STRING(msg, errors, naddrs); - gpr_free(msg); - - const char* warning_message = grpc_error_string(err); - gpr_log(GPR_INFO, "WARNING: %s", warning_message); - - /* we managed to bind some addresses: continue */ - } - grpc_resolved_addresses_destroy(resolved); - - arg = grpc_channel_args_find(args, GRPC_ARG_ENABLE_CHANNELZ); - if (grpc_channel_arg_get_bool(arg, GRPC_ENABLE_CHANNELZ_DEFAULT)) { - state->channelz_listen_socket = - grpc_core::MakeRefCounted( - addr, absl::StrFormat("chttp2 listener %s", addr)); - } - - /* Register with the server only upon success */ - grpc_server_add_listener(server, state, server_start_listener, - server_destroy_listener, - state->channelz_listen_socket); - goto done; - -/* Error path: cleanup and return */ -error: - GPR_ASSERT(err != GRPC_ERROR_NONE); - if (resolved) { - grpc_resolved_addresses_destroy(resolved); - } - if (tcp_server) { - grpc_tcp_server_unref(tcp_server); - } else { - grpc_channel_args_destroy(args); - gpr_free(state); - } - *port_num = 0; - -done: - if (errors != nullptr) { - for (i = 0; i < naddrs; i++) { - GRPC_ERROR_UNREF(errors[i]); - } - gpr_free(errors); - } - return err; + return grpc_core::Chttp2ServerListener::Create(server, addr, args, port_num); } + +} // namespace grpc_core diff --git a/src/core/ext/transport/chttp2/server/chttp2_server.h b/src/core/ext/transport/chttp2/server/chttp2_server.h index 6e51001b53b..8f651bb5ce9 100644 --- a/src/core/ext/transport/chttp2/server/chttp2_server.h +++ b/src/core/ext/transport/chttp2/server/chttp2_server.h @@ -25,9 +25,13 @@ #include "src/core/lib/iomgr/error.h" +namespace grpc_core { + /// Adds a port to \a server. Sets \a port_num to the port number. /// Takes ownership of \a args. -grpc_error* grpc_chttp2_server_add_port(grpc_server* server, const char* addr, - grpc_channel_args* args, int* port_num); +grpc_error* Chttp2ServerAddPort(grpc_server* server, const char* addr, + grpc_channel_args* args, int* port_num); + +} // namespace grpc_core #endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_SERVER_CHTTP2_SERVER_H */ diff --git a/src/core/ext/transport/chttp2/server/insecure/server_chttp2.cc b/src/core/ext/transport/chttp2/server/insecure/server_chttp2.cc index 99f18cdf39f..7220591a30f 100644 --- a/src/core/ext/transport/chttp2/server/insecure/server_chttp2.cc +++ b/src/core/ext/transport/chttp2/server/insecure/server_chttp2.cc @@ -32,7 +32,7 @@ int grpc_server_add_insecure_http2_port(grpc_server* server, const char* addr) { int port_num = 0; GRPC_API_TRACE("grpc_server_add_insecure_http2_port(server=%p, addr=%s)", 2, (server, addr)); - grpc_error* err = grpc_chttp2_server_add_port( + grpc_error* err = grpc_core::Chttp2ServerAddPort( server, addr, grpc_channel_args_copy(grpc_server_get_channel_args(server)), &port_num); if (err != GRPC_ERROR_NONE) { diff --git a/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.cc b/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.cc index 98fdb620704..6e203ed4e92 100644 --- a/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.cc +++ b/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.cc @@ -72,7 +72,7 @@ int grpc_server_add_secure_http2_port(grpc_server* server, const char* addr, grpc_channel_args_copy_and_add(grpc_server_get_channel_args(server), args_to_add, GPR_ARRAY_SIZE(args_to_add)); // Add server port. - err = grpc_chttp2_server_add_port(server, addr, args, &port_num); + err = grpc_core::Chttp2ServerAddPort(server, addr, args, &port_num); done: sc.reset(DEBUG_LOCATION, "server"); diff --git a/src/core/lib/surface/server.cc b/src/core/lib/surface/server.cc index d22b52ad6fe..6999704e138 100644 --- a/src/core/lib/surface/server.cc +++ b/src/core/lib/surface/server.cc @@ -24,12 +24,14 @@ #include #include +#include +#include +#include + #include #include #include -#include - #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/channel/channelz.h" #include "src/core/lib/channel/connected_channel.h" @@ -57,13 +59,12 @@ namespace { void server_on_recv_initial_metadata(void* ptr, grpc_error* error); void server_recv_trailing_metadata_ready(void* user_data, grpc_error* error); -struct listener { - void* arg; - void (*start)(grpc_server* server, void* arg, grpc_pollset** pollsets, - size_t pollset_count); - void (*destroy)(grpc_server* server, void* arg, grpc_closure* closure); - struct listener* next; - intptr_t socket_uuid; +struct Listener { + explicit Listener( + grpc_core::OrphanablePtr l) + : listener(std::move(l)) {} + + grpc_core::OrphanablePtr listener; grpc_closure destroy_done; }; @@ -297,15 +298,15 @@ struct channel_broadcaster { } // namespace struct grpc_server { - grpc_channel_args* channel_args; + grpc_channel_args* channel_args = nullptr; - grpc_resource_user* default_resource_user; + grpc_resource_user* default_resource_user = nullptr; - grpc_completion_queue** cqs; - grpc_pollset** pollsets; - size_t cq_count; - size_t pollset_count; - bool started; + grpc_completion_queue** cqs = nullptr; + grpc_pollset** pollsets = nullptr; + size_t cq_count = 0; + size_t pollset_count = 0; + bool started = false; /* The two following mutexes control access to server-state mu_global controls access to non-call-related state (e.g., channel state) @@ -319,26 +320,26 @@ struct grpc_server { /* startup synchronization: flag is protected by mu_global, signals whether we are doing the listener start routine or not */ - bool starting; + bool starting = false; gpr_cv starting_cv; // TODO(vjpai): Convert from a linked-list head pointer to a std::vector once // grpc_server has a real constructor/destructor - registered_method* registered_methods; + registered_method* registered_methods = nullptr; /** one request matcher for unregistered methods */ // TODO(vjpai): Convert to a std::unique_ptr once grpc_server has a real // constructor and destructor. - RequestMatcherInterface* unregistered_request_matcher; + RequestMatcherInterface* unregistered_request_matcher = nullptr; - gpr_atm shutdown_flag; - uint8_t shutdown_published; - size_t num_shutdown_tags; - shutdown_tag* shutdown_tags; + gpr_atm shutdown_flag = 0; + uint8_t shutdown_published = 0; + size_t num_shutdown_tags = 0; + shutdown_tag* shutdown_tags = nullptr; channel_data root_channel_data; - listener* listeners; - int listeners_destroyed; + std::list listeners; + int listeners_destroyed = 0; grpc_core::RefCount internal_refcount; /** when did we print the last shutdown progress message */ @@ -675,7 +676,6 @@ void server_ref(grpc_server* server) { server->internal_refcount.Ref(); } void server_delete(grpc_server* server) { registered_method* rm; size_t i; - server->channelz_server.reset(); grpc_channel_args_destroy(server->channel_args); gpr_mu_destroy(&server->mu_global); gpr_mu_destroy(&server->mu_call); @@ -691,7 +691,7 @@ void server_delete(grpc_server* server) { gpr_free(server->cqs); gpr_free(server->pollsets); gpr_free(server->shutdown_tags); - gpr_free(server); + delete server; } void server_unref(grpc_server* server) { @@ -881,15 +881,6 @@ void start_new_rpc(grpc_call_element* elem) { GRPC_SRM_PAYLOAD_NONE); } -int num_listeners(grpc_server* server) { - listener* l; - int n = 0; - for (l = server->listeners; l; l = l->next) { - n++; - } - return n; -} - void done_shutdown_event(void* server, grpc_cq_completion* /*completion*/) { server_unref(static_cast(server)); } @@ -929,17 +920,17 @@ void maybe_finish_shutdown(grpc_server* server) { gpr_mu_unlock(&server->mu_call); if (server->root_channel_data.next != &server->root_channel_data || - server->listeners_destroyed < num_listeners(server)) { + server->listeners_destroyed < server->listeners.size()) { if (gpr_time_cmp(gpr_time_sub(gpr_now(GPR_CLOCK_REALTIME), server->last_shutdown_message_time), gpr_time_from_seconds(1, GPR_TIMESPAN)) >= 0) { server->last_shutdown_message_time = gpr_now(GPR_CLOCK_REALTIME); gpr_log(GPR_DEBUG, - "Waiting for %d channels and %d/%d listeners to be destroyed" - " before shutting down server", + "Waiting for %d channels and %" PRIuPTR "/%" PRIuPTR + " listeners to be destroyed before shutting down server", num_channels(server), - num_listeners(server) - server->listeners_destroyed, - num_listeners(server)); + server->listeners.size() - server->listeners_destroyed, + server->listeners.size()); } return; } @@ -1312,26 +1303,21 @@ grpc_server* grpc_server_create(const grpc_channel_args* args, void* reserved) { grpc_core::ExecCtx exec_ctx; GRPC_API_TRACE("grpc_server_create(%p, %p)", 2, (args, reserved)); - grpc_server* server = - static_cast(gpr_zalloc(sizeof(grpc_server))); + grpc_server* server = new grpc_server; gpr_mu_init(&server->mu_global); gpr_mu_init(&server->mu_call); gpr_cv_init(&server->starting_cv); - /* decremented by grpc_server_destroy */ - new (&server->internal_refcount) grpc_core::RefCount(); server->root_channel_data.next = server->root_channel_data.prev = &server->root_channel_data; server->channel_args = grpc_channel_args_copy(args); - const grpc_arg* arg = grpc_channel_args_find(args, GRPC_ARG_ENABLE_CHANNELZ); - if (grpc_channel_arg_get_bool(arg, GRPC_ENABLE_CHANNELZ_DEFAULT)) { - arg = grpc_channel_args_find( - args, GRPC_ARG_MAX_CHANNEL_TRACE_EVENT_MEMORY_PER_NODE); - size_t channel_tracer_max_memory = grpc_channel_arg_get_integer( - arg, + if (grpc_channel_args_find_bool(args, GRPC_ARG_ENABLE_CHANNELZ, + GRPC_ENABLE_CHANNELZ_DEFAULT)) { + size_t channel_tracer_max_memory = grpc_channel_args_find_integer( + args, GRPC_ARG_MAX_CHANNEL_TRACE_EVENT_MEMORY_PER_NODE, {GRPC_MAX_CHANNEL_TRACE_EVENT_MEMORY_PER_NODE_DEFAULT, 0, INT_MAX}); server->channelz_server = grpc_core::MakeRefCounted( @@ -1414,8 +1400,8 @@ void grpc_server_start(grpc_server* server) { server->starting = true; gpr_mu_unlock(&server->mu_global); - for (listener* l = server->listeners; l; l = l->next) { - l->start(server, l->arg, server->pollsets, server->pollset_count); + for (auto& listener : server->listeners) { + listener.listener->Start(server, server->pollsets, server->pollset_count); } gpr_mu_lock(&server->mu_global); @@ -1547,7 +1533,6 @@ void grpc_server_setup_transport( */ void grpc_server_shutdown_and_notify(grpc_server* server, grpc_completion_queue* cq, void* tag) { - listener* l; shutdown_tag* sdt; channel_broadcaster broadcaster; grpc_core::ApplicationCallbackExecCtx callback_exec_ctx; @@ -1599,13 +1584,18 @@ void grpc_server_shutdown_and_notify(grpc_server* server, gpr_mu_unlock(&server->mu_global); /* Shutdown listeners */ - for (l = server->listeners; l; l = l->next) { - GRPC_CLOSURE_INIT(&l->destroy_done, listener_destroy_done, server, - grpc_schedule_on_exec_ctx); - l->destroy(server, l->arg, &l->destroy_done); - if (server->channelz_server != nullptr && l->socket_uuid != 0) { - server->channelz_server->RemoveChildListenSocket(l->socket_uuid); + for (auto& listener : server->listeners) { + grpc_core::channelz::ListenSocketNode* channelz_listen_socket_node = + listener.listener->channelz_listen_socket_node(); + if (server->channelz_server != nullptr && + channelz_listen_socket_node != nullptr) { + server->channelz_server->RemoveChildListenSocket( + channelz_listen_socket_node->uuid()); } + GRPC_CLOSURE_INIT(&listener.destroy_done, listener_destroy_done, server, + grpc_schedule_on_exec_ctx); + listener.listener->SetOnDestroyDone(&listener.destroy_done); + listener.listener.reset(); } channel_broadcaster_shutdown(&broadcaster, true /* send_goaway */, @@ -1636,46 +1626,29 @@ void grpc_server_cancel_all_calls(grpc_server* server) { } void grpc_server_destroy(grpc_server* server) { - listener* l; grpc_core::ApplicationCallbackExecCtx callback_exec_ctx; grpc_core::ExecCtx exec_ctx; GRPC_API_TRACE("grpc_server_destroy(server=%p)", 1, (server)); gpr_mu_lock(&server->mu_global); - GPR_ASSERT(gpr_atm_acq_load(&server->shutdown_flag) || !server->listeners); - GPR_ASSERT(server->listeners_destroyed == num_listeners(server)); - - while (server->listeners) { - l = server->listeners; - server->listeners = l->next; - gpr_free(l); - } - + GPR_ASSERT(gpr_atm_acq_load(&server->shutdown_flag) || + server->listeners.empty()); + GPR_ASSERT(server->listeners_destroyed == server->listeners.size()); gpr_mu_unlock(&server->mu_global); server_unref(server); } void grpc_server_add_listener( - grpc_server* server, void* listener_arg, - void (*start)(grpc_server* server, void* arg, grpc_pollset** pollsets, - size_t pollset_count), - void (*destroy)(grpc_server* server, void* arg, grpc_closure* on_done), - grpc_core::RefCountedPtr node) { - listener* l = static_cast(gpr_malloc(sizeof(listener))); - l->arg = listener_arg; - l->start = start; - l->destroy = destroy; - l->socket_uuid = 0; - if (node != nullptr) { - l->socket_uuid = node->uuid(); - if (server->channelz_server != nullptr) { - server->channelz_server->AddChildListenSocket(std::move(node)); - } - } - l->next = server->listeners; - server->listeners = l; + grpc_server* server, + grpc_core::OrphanablePtr listener) { + grpc_core::channelz::ListenSocketNode* listen_socket_node = + listener->channelz_listen_socket_node(); + if (listen_socket_node != nullptr && server->channelz_server != nullptr) { + server->channelz_server->AddChildListenSocket(listen_socket_node->Ref()); + } + server->listeners.emplace_back(std::move(listener)); } namespace { diff --git a/src/core/lib/surface/server.h b/src/core/lib/surface/server.h index 3f11c83caa9..04385e1d284 100644 --- a/src/core/lib/surface/server.h +++ b/src/core/lib/surface/server.h @@ -32,14 +32,35 @@ extern const grpc_channel_filter grpc_server_top_filter; /** Lightweight tracing of server channel state */ extern grpc_core::TraceFlag grpc_server_channel_trace; -/* Add a listener to the server: when the server starts, it will call start, - and when it shuts down, it will call destroy */ +namespace grpc_core { + +/// Interface for listeners. +/// Implementations must override the Orphan() method, which should stop +/// listening and initiate destruction of the listener. +class ServerListenerInterface : public Orphanable { + public: + virtual ~ServerListenerInterface() = default; + + /// Starts listening. + virtual void Start(grpc_server* server, grpc_pollset** pollsets, + size_t npollsets) = 0; + + /// Returns the channelz node for the listen socket, or null if not + /// supported. + virtual channelz::ListenSocketNode* channelz_listen_socket_node() const = 0; + + /// Sets a closure to be invoked by the listener when its destruction + /// is complete. + virtual void SetOnDestroyDone(grpc_closure* on_destroy_done) = 0; +}; + +} // namespace grpc_core + +/* Add a listener to the server: when the server starts, it will call Start(), + and when it shuts down, it will orphan the listener. */ void grpc_server_add_listener( - grpc_server* server, void* listener_arg, - void (*start)(grpc_server* server, void* arg, grpc_pollset** pollsets, - size_t npollsets), - void (*destroy)(grpc_server* server, void* arg, grpc_closure* on_done), - grpc_core::RefCountedPtr node); + grpc_server* server, + grpc_core::OrphanablePtr listener); /* Setup a transport - creates a channel stack, binds the transport to the server */ diff --git a/test/core/surface/BUILD b/test/core/surface/BUILD index 7d789393c91..91f42cca637 100644 --- a/test/core/surface/BUILD +++ b/test/core/surface/BUILD @@ -156,6 +156,9 @@ grpc_cc_test( grpc_cc_test( name = "server_chttp2_test", srcs = ["server_chttp2_test.cc"], + external_deps = [ + "gtest", + ], language = "C++", deps = [ "//:gpr", diff --git a/test/core/surface/server_chttp2_test.cc b/test/core/surface/server_chttp2_test.cc index 3399b07a1c0..e74b0e75617 100644 --- a/test/core/surface/server_chttp2_test.cc +++ b/test/core/surface/server_chttp2_test.cc @@ -16,6 +16,8 @@ * */ +#include + #include #include #include @@ -29,21 +31,19 @@ #include "test/core/util/port.h" #include "test/core/util/test_config.h" -void test_unparsable_target(void) { +TEST(ServerChttp2, UnparseableTarget) { grpc_channel_args args = {0, nullptr}; grpc_server* server = grpc_server_create(&args, nullptr); int port = grpc_server_add_insecure_http2_port(server, "["); - GPR_ASSERT(port == 0); + EXPECT_EQ(port, 0); grpc_server_destroy(server); } // GRPC_ARG_ALLOW_REUSEPORT isn't supported for custom servers #ifndef GRPC_UV -void test_add_same_port_twice() { - grpc_arg a; - a.type = GRPC_ARG_INTEGER; - a.key = const_cast(GRPC_ARG_ALLOW_REUSEPORT); - a.value.integer = 0; +TEST(ServerChttp2, AddSamePortTwice) { + grpc_arg a = grpc_channel_arg_integer_create( + const_cast(GRPC_ARG_ALLOW_REUSEPORT), 0); grpc_channel_args args = {1, &a}; int port = grpc_pick_unused_port_or_die(); @@ -52,10 +52,10 @@ void test_add_same_port_twice() { grpc_server_credentials* fake_creds = grpc_fake_transport_security_server_credentials_create(); std::string addr = grpc_core::JoinHostPort("localhost", port); - GPR_ASSERT( - grpc_server_add_secure_http2_port(server, addr.c_str(), fake_creds)); - GPR_ASSERT( - grpc_server_add_secure_http2_port(server, addr.c_str(), fake_creds) == 0); + EXPECT_EQ(grpc_server_add_secure_http2_port(server, addr.c_str(), fake_creds), + port); + EXPECT_EQ(grpc_server_add_secure_http2_port(server, addr.c_str(), fake_creds), + 0); grpc_server_credentials_release(fake_creds); grpc_server_shutdown_and_notify(server, cq, nullptr); @@ -68,11 +68,9 @@ void test_add_same_port_twice() { int main(int argc, char** argv) { grpc::testing::TestEnvironment env(argc, argv); + ::testing::InitGoogleTest(&argc, argv); grpc_init(); - test_unparsable_target(); -#ifndef GRPC_UV - test_add_same_port_twice(); -#endif + int ret = RUN_ALL_TESTS(); grpc_shutdown(); - return 0; + return ret; } diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index d111b2fd8d7..e5db6477cf4 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -2351,30 +2351,6 @@ ], "uses_polling": true }, - { - "args": [], - "benchmark": false, - "ci_platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "gtest": false, - "language": "c", - "name": "server_chttp2_test", - "platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "uses_polling": true - }, { "args": [], "benchmark": false, @@ -5239,6 +5215,30 @@ ], "uses_polling": true }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": true, + "language": "c++", + "name": "server_chttp2_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "uses_polling": true + }, { "args": [], "benchmark": false, From b3469c723eaf7ee3bd489181b7ce555d8981c57c Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Fri, 19 Jun 2020 13:49:16 -0700 Subject: [PATCH 76/80] Pre-assigned the server port and added more logs --- test/cpp/qps/json_run_localhost.cc | 10 +++++++--- test/cpp/qps/qps_worker.cc | 6 +++--- test/cpp/qps/server_async.cc | 8 +++++++- test/cpp/qps/server_callback.cc | 8 +++++++- test/cpp/qps/server_sync.cc | 8 +++++++- 5 files changed, 31 insertions(+), 9 deletions(-) diff --git a/test/cpp/qps/json_run_localhost.cc b/test/cpp/qps/json_run_localhost.cc index e7383a245ba..eaa6e4a1b5f 100644 --- a/test/cpp/qps/json_run_localhost.cc +++ b/test/cpp/qps/json_run_localhost.cc @@ -88,12 +88,16 @@ int main(int argc, char** argv) { bool first = true; for (int i = 0; i < kNumWorkers; i++) { - const auto port = grpc_pick_unused_port_or_die(); + const auto driver_port = grpc_pick_unused_port_or_die(); + // ServerPort can be used or not later depending on the type of worker + // but we like to issue all ports required here to avoid port conflict. + const auto server_port = grpc_pick_unused_port_or_die(); std::vector args = {bin_dir + "/qps_worker", "-driver_port", - as_string(port)}; + as_string(driver_port), "-server_port", + as_string(server_port)}; g_workers[i] = new SubProcess(args); if (!first) env << ","; - env << "localhost:" << port; + env << "localhost:" << driver_port; first = false; } diff --git a/test/cpp/qps/qps_worker.cc b/test/cpp/qps/qps_worker.cc index fd207765ca5..5d6a2fa1f1a 100644 --- a/test/cpp/qps/qps_worker.cc +++ b/test/cpp/qps/qps_worker.cc @@ -280,6 +280,7 @@ QpsWorker::QpsWorker(int driver_port, int server_port, gpr_atm_rel_store(&done_, static_cast(0)); std::unique_ptr builder = CreateQpsServerBuilder(); + builder->AddChannelArgument(GRPC_ARG_ALLOW_REUSEPORT, 0); if (driver_port >= 0) { std::string server_address = grpc_core::JoinHostPort("::", driver_port); builder->AddListeningPort( @@ -291,11 +292,10 @@ QpsWorker::QpsWorker(int driver_port, int server_port, server_ = builder->BuildAndStart(); if (server_ == nullptr) { gpr_log(GPR_ERROR, - "QpsWorker: Fail to BuildAndStart(driver_port=%d, server_port=%d)", + "QpsWorker: Fail to BuildAndStart(port=%d) (server_port=%d)", driver_port, server_port); } else { - gpr_log(GPR_INFO, - "QpsWorker: BuildAndStart(driver_port=%d, server_port=%d) done", + gpr_log(GPR_INFO, "QpsWorker: BuildAndStart(port=%d) (server_port=%d)", driver_port, server_port); } } diff --git a/test/cpp/qps/server_async.cc b/test/cpp/qps/server_async.cc index 6a07accb151..51d410b1cbf 100644 --- a/test/cpp/qps/server_async.cc +++ b/test/cpp/qps/server_async.cc @@ -82,7 +82,8 @@ class AsyncQpsServerTest final : public grpc::testing::Server { if (port_num >= 0) { std::string server_address = grpc_core::JoinHostPort("::", port_num); builder->AddListeningPort(server_address.c_str(), - Server::CreateServerCredentials(config)); + Server::CreateServerCredentials(config), + &port_num); } register_service(builder.get(), &async_service_); @@ -105,6 +106,11 @@ class AsyncQpsServerTest final : public grpc::testing::Server { ApplyConfigToBuilder(config, builder.get()); server_ = builder->BuildAndStart(); + if (server_ == nullptr) { + gpr_log(GPR_ERROR, "Server: Fail to BuildAndStart(port=%d)", port_num); + } else { + gpr_log(GPR_INFO, "Server: BuildAndStart(port=%d)", port_num); + } auto process_rpc_bound = std::bind(process_rpc, config.payload_config(), std::placeholders::_1, diff --git a/test/cpp/qps/server_callback.cc b/test/cpp/qps/server_callback.cc index e4aa10946d2..a0a95312229 100644 --- a/test/cpp/qps/server_callback.cc +++ b/test/cpp/qps/server_callback.cc @@ -104,7 +104,8 @@ class CallbackServer final : public grpc::testing::Server { if (port_num >= 0) { std::string server_address = grpc_core::JoinHostPort("::", port_num); builder->AddListeningPort(server_address.c_str(), - Server::CreateServerCredentials(config)); + Server::CreateServerCredentials(config), + &port_num); } ApplyConfigToBuilder(config, builder.get()); @@ -112,6 +113,11 @@ class CallbackServer final : public grpc::testing::Server { builder->RegisterService(&service_); impl_ = builder->BuildAndStart(); + if (impl_ == nullptr) { + gpr_log(GPR_ERROR, "Server: Fail to BuildAndStart(port=%d)", port_num); + } else { + gpr_log(GPR_INFO, "Server: BuildAndStart(port=%d)", port_num); + } } std::shared_ptr InProcessChannel( diff --git a/test/cpp/qps/server_sync.cc b/test/cpp/qps/server_sync.cc index 19d406a167c..13b3c248ac2 100644 --- a/test/cpp/qps/server_sync.cc +++ b/test/cpp/qps/server_sync.cc @@ -162,7 +162,8 @@ class SynchronousServer final : public grpc::testing::Server { if (port_num >= 0) { std::string server_address = grpc_core::JoinHostPort("::", port_num); builder->AddListeningPort(server_address.c_str(), - Server::CreateServerCredentials(config)); + Server::CreateServerCredentials(config), + &port_num); } ApplyConfigToBuilder(config, builder.get()); @@ -170,6 +171,11 @@ class SynchronousServer final : public grpc::testing::Server { builder->RegisterService(&service_); impl_ = builder->BuildAndStart(); + if (impl_ == nullptr) { + gpr_log(GPR_ERROR, "Server: Fail to BuildAndStart(port=%d)", port_num); + } else { + gpr_log(GPR_INFO, "Server: BuildAndStart(port=%d)", port_num); + } } std::shared_ptr InProcessChannel( From 0497ad8bb0c02995f16a76b2ce85753a026f08bc Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Wed, 10 Jun 2020 15:25:40 -0700 Subject: [PATCH 77/80] Add missing message-size check before decompressing Add and fix tests for limit check before decompression Code restructuring to allow easy reuse of service config call data Regenerate projects Reviewer comments --- BUILD | 2 + BUILD.gn | 1 + CMakeLists.txt | 2 + Makefile | 2 + build_autogenerated.yaml | 2 + config.m4 | 1 + config.w32 | 1 + gRPC-Core.podspec | 1 + grpc.gemspec | 1 + grpc.gyp | 2 + package.xml | 1 + .../service_config_channel_arg_filter.cc | 142 ++++++++ .../ext/filters/http/http_filters_plugin.cc | 3 +- .../message_decompress_filter.cc | 92 +++-- .../message_decompress_filter.h | 4 +- .../message_size/message_size_filter.cc | 106 +++--- .../message_size/message_size_filter.h | 6 + .../plugin_registry/grpc_plugin_registry.cc | 4 + .../grpc_unsecure_plugin_registry.cc | 4 + src/python/grpcio/grpc_core_dependencies.py | 1 + test/core/end2end/tests/max_message_length.cc | 332 ++++++++++++++++++ test/cpp/microbenchmarks/bm_call_create.cc | 3 +- tools/doxygen/Doxyfile.c++.internal | 1 + tools/doxygen/Doxyfile.core.internal | 1 + 24 files changed, 622 insertions(+), 93 deletions(-) create mode 100644 src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc diff --git a/BUILD b/BUILD index 0d2a7ca058c..bbb1d8dfeea 100644 --- a/BUILD +++ b/BUILD @@ -1044,6 +1044,7 @@ grpc_cc_library( "src/core/ext/filters/client_channel/retry_throttle.cc", "src/core/ext/filters/client_channel/server_address.cc", "src/core/ext/filters/client_channel/service_config.cc", + "src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc", "src/core/ext/filters/client_channel/service_config_parser.cc", "src/core/ext/filters/client_channel/subchannel.cc", "src/core/ext/filters/client_channel/subchannel_pool_interface.cc", @@ -1184,6 +1185,7 @@ grpc_cc_library( language = "c++", deps = [ "grpc_base", + "grpc_message_size_filter", ], ) diff --git a/BUILD.gn b/BUILD.gn index a16d298f1be..5e5f8622b34 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -293,6 +293,7 @@ config("grpc_config") { "src/core/ext/filters/client_channel/service_config.cc", "src/core/ext/filters/client_channel/service_config.h", "src/core/ext/filters/client_channel/service_config_call_data.h", + "src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc", "src/core/ext/filters/client_channel/service_config_parser.cc", "src/core/ext/filters/client_channel/service_config_parser.h", "src/core/ext/filters/client_channel/subchannel.cc", diff --git a/CMakeLists.txt b/CMakeLists.txt index 323f7800627..730dc9b0775 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1375,6 +1375,7 @@ add_library(grpc src/core/ext/filters/client_channel/retry_throttle.cc src/core/ext/filters/client_channel/server_address.cc src/core/ext/filters/client_channel/service_config.cc + src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc src/core/ext/filters/client_channel/service_config_parser.cc src/core/ext/filters/client_channel/subchannel.cc src/core/ext/filters/client_channel/subchannel_pool_interface.cc @@ -2046,6 +2047,7 @@ add_library(grpc_unsecure src/core/ext/filters/client_channel/retry_throttle.cc src/core/ext/filters/client_channel/server_address.cc src/core/ext/filters/client_channel/service_config.cc + src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc src/core/ext/filters/client_channel/service_config_parser.cc src/core/ext/filters/client_channel/subchannel.cc src/core/ext/filters/client_channel/subchannel_pool_interface.cc diff --git a/Makefile b/Makefile index bbaccc44513..2de807dead0 100644 --- a/Makefile +++ b/Makefile @@ -3677,6 +3677,7 @@ LIBGRPC_SRC = \ src/core/ext/filters/client_channel/retry_throttle.cc \ src/core/ext/filters/client_channel/server_address.cc \ src/core/ext/filters/client_channel/service_config.cc \ + src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc \ src/core/ext/filters/client_channel/service_config_parser.cc \ src/core/ext/filters/client_channel/subchannel.cc \ src/core/ext/filters/client_channel/subchannel_pool_interface.cc \ @@ -4322,6 +4323,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/ext/filters/client_channel/retry_throttle.cc \ src/core/ext/filters/client_channel/server_address.cc \ src/core/ext/filters/client_channel/service_config.cc \ + src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc \ src/core/ext/filters/client_channel/service_config_parser.cc \ src/core/ext/filters/client_channel/subchannel.cc \ src/core/ext/filters/client_channel/subchannel_pool_interface.cc \ diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index 9eb53b8e576..57ee5eabde8 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -793,6 +793,7 @@ libs: - src/core/ext/filters/client_channel/retry_throttle.cc - src/core/ext/filters/client_channel/server_address.cc - src/core/ext/filters/client_channel/service_config.cc + - src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc - src/core/ext/filters/client_channel/service_config_parser.cc - src/core/ext/filters/client_channel/subchannel.cc - src/core/ext/filters/client_channel/subchannel_pool_interface.cc @@ -1652,6 +1653,7 @@ libs: - src/core/ext/filters/client_channel/retry_throttle.cc - src/core/ext/filters/client_channel/server_address.cc - src/core/ext/filters/client_channel/service_config.cc + - src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc - src/core/ext/filters/client_channel/service_config_parser.cc - src/core/ext/filters/client_channel/subchannel.cc - src/core/ext/filters/client_channel/subchannel_pool_interface.cc diff --git a/config.m4 b/config.m4 index cd1019abcd9..4f0fd8513d9 100644 --- a/config.m4 +++ b/config.m4 @@ -92,6 +92,7 @@ if test "$PHP_GRPC" != "no"; then src/core/ext/filters/client_channel/retry_throttle.cc \ src/core/ext/filters/client_channel/server_address.cc \ src/core/ext/filters/client_channel/service_config.cc \ + src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc \ src/core/ext/filters/client_channel/service_config_parser.cc \ src/core/ext/filters/client_channel/subchannel.cc \ src/core/ext/filters/client_channel/subchannel_pool_interface.cc \ diff --git a/config.w32 b/config.w32 index 8109f9103af..093add492ad 100644 --- a/config.w32 +++ b/config.w32 @@ -61,6 +61,7 @@ if (PHP_GRPC != "no") { "src\\core\\ext\\filters\\client_channel\\retry_throttle.cc " + "src\\core\\ext\\filters\\client_channel\\server_address.cc " + "src\\core\\ext\\filters\\client_channel\\service_config.cc " + + "src\\core\\ext\\filters\\client_channel\\service_config_channel_arg_filter.cc " + "src\\core\\ext\\filters\\client_channel\\service_config_parser.cc " + "src\\core\\ext\\filters\\client_channel\\subchannel.cc " + "src\\core\\ext\\filters\\client_channel\\subchannel_pool_interface.cc " + diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 695768422c2..e4ae503d0a2 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -277,6 +277,7 @@ Pod::Spec.new do |s| 'src/core/ext/filters/client_channel/service_config.cc', 'src/core/ext/filters/client_channel/service_config.h', 'src/core/ext/filters/client_channel/service_config_call_data.h', + 'src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc', 'src/core/ext/filters/client_channel/service_config_parser.cc', 'src/core/ext/filters/client_channel/service_config_parser.h', 'src/core/ext/filters/client_channel/subchannel.cc', diff --git a/grpc.gemspec b/grpc.gemspec index af04b9d9de4..66abcc3f64f 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -199,6 +199,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/filters/client_channel/service_config.cc ) s.files += %w( src/core/ext/filters/client_channel/service_config.h ) s.files += %w( src/core/ext/filters/client_channel/service_config_call_data.h ) + s.files += %w( src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc ) s.files += %w( src/core/ext/filters/client_channel/service_config_parser.cc ) s.files += %w( src/core/ext/filters/client_channel/service_config_parser.h ) s.files += %w( src/core/ext/filters/client_channel/subchannel.cc ) diff --git a/grpc.gyp b/grpc.gyp index b80efd83534..bc1a11e3cca 100644 --- a/grpc.gyp +++ b/grpc.gyp @@ -487,6 +487,7 @@ 'src/core/ext/filters/client_channel/retry_throttle.cc', 'src/core/ext/filters/client_channel/server_address.cc', 'src/core/ext/filters/client_channel/service_config.cc', + 'src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc', 'src/core/ext/filters/client_channel/service_config_parser.cc', 'src/core/ext/filters/client_channel/subchannel.cc', 'src/core/ext/filters/client_channel/subchannel_pool_interface.cc', @@ -994,6 +995,7 @@ 'src/core/ext/filters/client_channel/retry_throttle.cc', 'src/core/ext/filters/client_channel/server_address.cc', 'src/core/ext/filters/client_channel/service_config.cc', + 'src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc', 'src/core/ext/filters/client_channel/service_config_parser.cc', 'src/core/ext/filters/client_channel/subchannel.cc', 'src/core/ext/filters/client_channel/subchannel_pool_interface.cc', diff --git a/package.xml b/package.xml index 0166a778789..93b22c15c0b 100644 --- a/package.xml +++ b/package.xml @@ -179,6 +179,7 @@ + diff --git a/src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc b/src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc new file mode 100644 index 00000000000..9aad7c71307 --- /dev/null +++ b/src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc @@ -0,0 +1,142 @@ +// +// Copyright 2020 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// This filter reads GRPC_ARG_SERVICE_CONFIG and populates ServiceConfigCallData +// in the call context per call for direct channels. + +#include + +#include "src/core/ext/filters/client_channel/service_config_call_data.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/channel/channel_stack.h" +#include "src/core/lib/channel/channel_stack_builder.h" +#include "src/core/lib/surface/channel_init.h" + +namespace grpc_core { + +namespace { + +class ServiceConfigChannelArgCallData {}; + +class ServiceConfigChannelArgChannelData { + public: + explicit ServiceConfigChannelArgChannelData( + const grpc_channel_element_args* args) { + const char* service_config_str = grpc_channel_arg_find_string( + args->channel_args, GRPC_ARG_SERVICE_CONFIG); + if (service_config_str != nullptr) { + grpc_error* service_config_error = GRPC_ERROR_NONE; + auto svc_cfg = grpc_core::ServiceConfig::Create(service_config_str, + &service_config_error); + if (service_config_error == GRPC_ERROR_NONE) { + svc_cfg_ = std::move(svc_cfg); + } else { + gpr_log(GPR_ERROR, "%s", grpc_error_string(service_config_error)); + } + GRPC_ERROR_UNREF(service_config_error); + } + } + + grpc_core::RefCountedPtr svc_cfg() const { + return svc_cfg_; + } + + private: + grpc_core::RefCountedPtr svc_cfg_; +}; + +grpc_error* ServiceConfigChannelArgInitCallElem( + grpc_call_element* elem, const grpc_call_element_args* args) { + ServiceConfigChannelArgChannelData* chand = + static_cast(elem->channel_data); + if (chand->svc_cfg() != nullptr) { + GPR_DEBUG_ASSERT(args->context != nullptr); + args->arena->New( + chand->svc_cfg(), + chand->svc_cfg()->GetMethodParsedConfigVector(args->path), + args->context); + } + return GRPC_ERROR_NONE; +} + +void ServiceConfigChannelArgDestroyCallElem( + grpc_call_element* /* elem */, const grpc_call_final_info* /* final_info */, + grpc_closure* /* then_schedule_closure */) {} + +grpc_error* ServiceConfigChannelArgInitChannelElem( + grpc_channel_element* elem, grpc_channel_element_args* args) { + ServiceConfigChannelArgChannelData* chand = + static_cast(elem->channel_data); + new (chand) ServiceConfigChannelArgChannelData(args); + const char* service_config_str = grpc_channel_arg_get_string( + grpc_channel_args_find(args->channel_args, GRPC_ARG_SERVICE_CONFIG)); + if (service_config_str != nullptr) { + grpc_error* service_config_error = GRPC_ERROR_NONE; + auto svc_cfg = grpc_core::ServiceConfig::Create(service_config_str, + &service_config_error); + if (service_config_error == GRPC_ERROR_NONE) { + chand->svc_cfg = std::move(svc_cfg); + } else { + gpr_log(GPR_ERROR, "%s", grpc_error_string(service_config_error)); + } + GRPC_ERROR_UNREF(service_config_error); + } + return GRPC_ERROR_NONE; +} + +void ServiceConfigChannelArgDestroyChannelElem(grpc_channel_element* elem) { + ServiceConfigChannelArgChannelData* chand = + static_cast(elem->channel_data); + chand->~ServiceConfigChannelArgChannelData(); +} + +const grpc_channel_filter ServiceConfigChannelArgFilter = { + grpc_call_next_op, + grpc_channel_next_op, + sizeof(ServiceConfigChannelArgCallData), + ServiceConfigChannelArgInitCallElem, + grpc_call_stack_ignore_set_pollset_or_pollset_set, + ServiceConfigChannelArgDestroyCallElem, + sizeof(ServiceConfigChannelArgChannelData), + ServiceConfigChannelArgInitChannelElem, + ServiceConfigChannelArgDestroyChannelElem, + grpc_channel_next_get_info, + "service_config_channel_arg"}; + +bool maybe_add_service_config_channel_arg_filter( + grpc_channel_stack_builder* builder, void* /* arg */) { + const grpc_channel_args* channel_args = + grpc_channel_stack_builder_get_channel_arguments(builder); + if (grpc_channel_args_want_minimal_stack(channel_args) || + grpc_channel_arg_get_string(grpc_channel_args_find( + channel_args, GRPC_ARG_SERVICE_CONFIG)) == nullptr) { + return true; + } + return grpc_channel_stack_builder_prepend_filter( + builder, &ServiceConfigChannelArgFilter, nullptr, nullptr); +} + +} // namespace + +} // namespace grpc_core + +void grpc_service_config_channel_arg_filter_init(void) { + grpc_channel_init_register_stage( + GRPC_CLIENT_DIRECT_CHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, + grpc_core::maybe_add_service_config_channel_arg_filter, nullptr); +} + +void grpc_service_config_channel_arg_filter_shutdown(void) {} diff --git a/src/core/ext/filters/http/http_filters_plugin.cc b/src/core/ext/filters/http/http_filters_plugin.cc index 2fedc7fe3d7..637dc3030f2 100644 --- a/src/core/ext/filters/http/http_filters_plugin.cc +++ b/src/core/ext/filters/http/http_filters_plugin.cc @@ -38,7 +38,8 @@ static optional_filter compress_filter = { &grpc_message_compress_filter, GRPC_ARG_ENABLE_PER_MESSAGE_COMPRESSION}; static optional_filter decompress_filter = { - &grpc_message_decompress_filter, GRPC_ARG_ENABLE_PER_MESSAGE_DECOMPRESSION}; + &grpc_core::MessageDecompressFilter, + GRPC_ARG_ENABLE_PER_MESSAGE_DECOMPRESSION}; static bool is_building_http_like_transport( grpc_channel_stack_builder* builder) { diff --git a/src/core/ext/filters/http/message_compress/message_decompress_filter.cc b/src/core/ext/filters/http/message_compress/message_decompress_filter.cc index d12f4013bb2..122c74e1e98 100644 --- a/src/core/ext/filters/http/message_compress/message_decompress_filter.cc +++ b/src/core/ext/filters/http/message_compress/message_decompress_filter.cc @@ -18,6 +18,8 @@ #include +#include "src/core/ext/filters/http/message_compress/message_decompress_filter.h" + #include #include @@ -27,7 +29,8 @@ #include #include -#include "src/core/ext/filters/http/message_compress/message_decompress_filter.h" +#include "absl/strings/str_format.h" +#include "src/core/ext/filters/message_size/message_size_filter.h" #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/compression/algorithm_metadata.h" #include "src/core/lib/compression/compression_args.h" @@ -37,14 +40,25 @@ #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/slice/slice_string_helpers.h" +namespace grpc_core { namespace { -class ChannelData {}; +class ChannelData { + public: + explicit ChannelData(const grpc_channel_element_args* args) + : max_recv_size_(get_max_recv_size(args->channel_args)) {} + + int max_recv_size() const { return max_recv_size_; } + + private: + int max_recv_size_; +}; class CallData { public: - explicit CallData(const grpc_call_element_args& args) - : call_combiner_(args.call_combiner) { + CallData(const grpc_call_element_args& args, const ChannelData* chand) + : call_combiner_(args.call_combiner), + max_recv_message_length_(chand->max_recv_size()) { // Initialize state for recv_initial_metadata_ready callback GRPC_CLOSURE_INIT(&on_recv_initial_metadata_ready_, OnRecvInitialMetadataReady, this, @@ -59,6 +73,13 @@ class CallData { GRPC_CLOSURE_INIT(&on_recv_trailing_metadata_ready_, OnRecvTrailingMetadataReady, this, grpc_schedule_on_exec_ctx); + const MessageSizeParsedConfig* limits = + get_message_size_config_from_call_context(args.context); + if (limits != nullptr && limits->limits().max_recv_size >= 0 && + (limits->limits().max_recv_size < max_recv_message_length_ || + max_recv_message_length_ < 0)) { + max_recv_message_length_ = limits->limits().max_recv_size; + } } ~CallData() { grpc_slice_buffer_destroy_internal(&recv_slices_); } @@ -82,7 +103,7 @@ class CallData { void MaybeResumeOnRecvTrailingMetadataReady(); static void OnRecvTrailingMetadataReady(void* arg, grpc_error* error); - grpc_core::CallCombiner* call_combiner_; + CallCombiner* call_combiner_; // Overall error for the call grpc_error* error_ = GRPC_ERROR_NONE; // Fields for handling recv_initial_metadata_ready callback @@ -91,17 +112,18 @@ class CallData { grpc_metadata_batch* recv_initial_metadata_ = nullptr; // Fields for handling recv_message_ready callback bool seen_recv_message_ready_ = false; + int max_recv_message_length_; grpc_message_compression_algorithm algorithm_ = GRPC_MESSAGE_COMPRESS_NONE; grpc_closure on_recv_message_ready_; grpc_closure* original_recv_message_ready_ = nullptr; grpc_closure on_recv_message_next_done_; - grpc_core::OrphanablePtr* recv_message_ = nullptr; + OrphanablePtr* recv_message_ = nullptr; // recv_slices_ holds the slices read from the original recv_message stream. // It is initialized during construction and reset when a new stream is // created using it. grpc_slice_buffer recv_slices_; - std::aligned_storage::type + std::aligned_storage::type recv_replacement_stream_; // Fields for handling recv_trailing_metadata_ready callback bool seen_recv_trailing_metadata_ready_ = false; @@ -139,7 +161,7 @@ void CallData::OnRecvInitialMetadataReady(void* arg, grpc_error* error) { calld->MaybeResumeOnRecvTrailingMetadataReady(); grpc_closure* closure = calld->original_recv_initial_metadata_ready_; calld->original_recv_initial_metadata_ready_ = nullptr; - grpc_core::Closure::Run(DEBUG_LOCATION, closure, GRPC_ERROR_REF(error)); + Closure::Run(DEBUG_LOCATION, closure, GRPC_ERROR_REF(error)); } void CallData::MaybeResumeOnRecvMessageReady() { @@ -170,6 +192,19 @@ void CallData::OnRecvMessageReady(void* arg, grpc_error* error) { 0) { return calld->ContinueRecvMessageReadyCallback(GRPC_ERROR_NONE); } + if (calld->max_recv_message_length_ >= 0 && + (*calld->recv_message_)->length() > + static_cast(calld->max_recv_message_length_)) { + std::string message_string = absl::StrFormat( + "Received message larger than max (%u vs. %d)", + (*calld->recv_message_)->length(), calld->max_recv_message_length_); + GPR_DEBUG_ASSERT(calld->error_ == GRPC_ERROR_NONE); + calld->error_ = grpc_error_set_int( + GRPC_ERROR_CREATE_FROM_COPIED_STRING(message_string.c_str()), + GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_RESOURCE_EXHAUSTED); + return calld->ContinueRecvMessageReadyCallback( + GRPC_ERROR_REF(calld->error_)); + } grpc_slice_buffer_destroy_internal(&calld->recv_slices_); grpc_slice_buffer_init(&calld->recv_slices_); return calld->ContinueReadingRecvMessage(); @@ -241,9 +276,9 @@ void CallData::FinishRecvMessage() { // Initializing recv_replacement_stream_ with decompressed_slices removes // all the slices from decompressed_slices leaving it empty. new (&recv_replacement_stream_) - grpc_core::SliceBufferByteStream(&decompressed_slices, recv_flags); - recv_message_->reset(reinterpret_cast( - &recv_replacement_stream_)); + SliceBufferByteStream(&decompressed_slices, recv_flags); + recv_message_->reset( + reinterpret_cast(&recv_replacement_stream_)); recv_message_ = nullptr; } ContinueRecvMessageReadyCallback(GRPC_ERROR_REF(error_)); @@ -254,7 +289,7 @@ void CallData::ContinueRecvMessageReadyCallback(grpc_error* error) { // The surface will clean up the receiving stream if there is an error. grpc_closure* closure = original_recv_message_ready_; original_recv_message_ready_ = nullptr; - grpc_core::Closure::Run(DEBUG_LOCATION, closure, error); + Closure::Run(DEBUG_LOCATION, closure, error); } void CallData::MaybeResumeOnRecvTrailingMetadataReady() { @@ -283,7 +318,7 @@ void CallData::OnRecvTrailingMetadataReady(void* arg, grpc_error* error) { calld->error_ = GRPC_ERROR_NONE; grpc_closure* closure = calld->original_recv_trailing_metadata_ready_; calld->original_recv_trailing_metadata_ready_ = nullptr; - grpc_core::Closure::Run(DEBUG_LOCATION, closure, error); + Closure::Run(DEBUG_LOCATION, closure, error); } void CallData::DecompressStartTransportStreamOpBatch( @@ -322,37 +357,44 @@ void DecompressStartTransportStreamOpBatch( calld->DecompressStartTransportStreamOpBatch(elem, batch); } -static grpc_error* DecompressInitCallElem(grpc_call_element* elem, - const grpc_call_element_args* args) { - new (elem->call_data) CallData(*args); +grpc_error* DecompressInitCallElem(grpc_call_element* elem, + const grpc_call_element_args* args) { + ChannelData* chand = static_cast(elem->channel_data); + new (elem->call_data) CallData(*args, chand); return GRPC_ERROR_NONE; } -static void DecompressDestroyCallElem( - grpc_call_element* elem, const grpc_call_final_info* /*final_info*/, - grpc_closure* /*ignored*/) { +void DecompressDestroyCallElem(grpc_call_element* elem, + const grpc_call_final_info* /*final_info*/, + grpc_closure* /*ignored*/) { CallData* calld = static_cast(elem->call_data); calld->~CallData(); } -static grpc_error* DecompressInitChannelElem( - grpc_channel_element* /*elem*/, grpc_channel_element_args* /*args*/) { +grpc_error* DecompressInitChannelElem(grpc_channel_element* elem, + grpc_channel_element_args* args) { + ChannelData* chand = static_cast(elem->channel_data); + new (chand) ChannelData(args); return GRPC_ERROR_NONE; } -void DecompressDestroyChannelElem(grpc_channel_element* /*elem*/) {} +void DecompressDestroyChannelElem(grpc_channel_element* elem) { + ChannelData* chand = static_cast(elem->channel_data); + chand->~ChannelData(); +} } // namespace -const grpc_channel_filter grpc_message_decompress_filter = { +const grpc_channel_filter MessageDecompressFilter = { DecompressStartTransportStreamOpBatch, grpc_channel_next_op, sizeof(CallData), DecompressInitCallElem, grpc_call_stack_ignore_set_pollset_or_pollset_set, DecompressDestroyCallElem, - 0, // sizeof(ChannelData) + sizeof(ChannelData), DecompressInitChannelElem, DecompressDestroyChannelElem, grpc_channel_next_get_info, "message_decompress"}; +} // namespace grpc_core diff --git a/src/core/ext/filters/http/message_compress/message_decompress_filter.h b/src/core/ext/filters/http/message_compress/message_decompress_filter.h index 7d567bf08a2..f19a4ca0cbd 100644 --- a/src/core/ext/filters/http/message_compress/message_decompress_filter.h +++ b/src/core/ext/filters/http/message_compress/message_decompress_filter.h @@ -23,7 +23,9 @@ #include "src/core/lib/channel/channel_stack.h" -extern const grpc_channel_filter grpc_message_decompress_filter; +namespace grpc_core { +extern const grpc_channel_filter MessageDecompressFilter; +} // namespace grpc_core #endif /* GRPC_CORE_EXT_FILTERS_HTTP_MESSAGE_COMPRESS_MESSAGE_DECOMPRESS_FILTER_H \ */ diff --git a/src/core/ext/filters/message_size/message_size_filter.cc b/src/core/ext/filters/message_size/message_size_filter.cc index d2ef5477636..53b4da451f5 100644 --- a/src/core/ext/filters/message_size/message_size_filter.cc +++ b/src/core/ext/filters/message_size/message_size_filter.cc @@ -45,6 +45,25 @@ namespace { size_t g_message_size_parser_index; } // namespace +// +// MessageSizeParsedConfig +// + +const MessageSizeParsedConfig* MessageSizeParsedConfig::GetFromCallContext( + const grpc_call_context_element* context) { + if (context == nullptr) return nullptr; + auto* svc_cfg_call_data = static_cast( + context[GRPC_CONTEXT_SERVICE_CONFIG_CALL_DATA].value); + if (svc_cfg_call_data == nullptr) return nullptr; + return static_cast( + svc_cfg_call_data->GetMethodParsedConfig( + MessageSizeParser::ParserIndex())); +} + +// +// MessageSizeParser +// + std::unique_ptr MessageSizeParser::ParsePerMethodParams(const Json& json, grpc_error** error) { GPR_DEBUG_ASSERT(error != nullptr && *error == GRPC_ERROR_NONE); @@ -97,12 +116,26 @@ void MessageSizeParser::Register() { } size_t MessageSizeParser::ParserIndex() { return g_message_size_parser_index; } + +int GetMaxRecvSizeFromChannelArgs(const grpc_channel_args* args) { + if (grpc_channel_args_want_minimal_stack(args)) return -1; + return grpc_channel_args_find_integer( + args, GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH, + {GRPC_DEFAULT_MAX_RECV_MESSAGE_LENGTH, -1, INT_MAX}); +} + +int GetMaxSendSizeFromChannelArgs(const grpc_channel_args* args) { + if (grpc_channel_args_want_minimal_stack(args)) return -1; + return grpc_channel_args_find_integer( + args, GRPC_ARG_MAX_SEND_MESSAGE_LENGTH, + {GRPC_DEFAULT_MAX_SEND_MESSAGE_LENGTH, -1, INT_MAX}); +} + } // namespace grpc_core namespace { struct channel_data { grpc_core::MessageSizeParsedConfig::message_size_limits limits; - grpc_core::RefCountedPtr svc_cfg; }; struct call_data { @@ -118,24 +151,8 @@ struct call_data { // Note: Per-method config is only available on the client, so we // apply the max request size to the send limit and the max response // size to the receive limit. - const grpc_core::MessageSizeParsedConfig* limits = nullptr; - grpc_core::ServiceConfigCallData* svc_cfg_call_data = nullptr; - if (args.context != nullptr) { - svc_cfg_call_data = static_cast( - args.context[GRPC_CONTEXT_SERVICE_CONFIG_CALL_DATA].value); - } - if (svc_cfg_call_data != nullptr) { - limits = static_cast( - svc_cfg_call_data->GetMethodParsedConfig( - grpc_core::MessageSizeParser::ParserIndex())); - } else if (chand.svc_cfg != nullptr) { - const auto* objs_vector = - chand.svc_cfg->GetMethodParsedConfigVector(args.path); - if (objs_vector != nullptr) { - limits = static_cast( - (*objs_vector)[grpc_core::MessageSizeParser::ParserIndex()].get()); - } - } + const grpc_core::MessageSizeParsedConfig* limits = + grpc_core::get_message_size_config_from_call_context(args.context); if (limits != nullptr) { if (limits->limits().max_send_size >= 0 && (limits->limits().max_send_size < this->limits.max_send_size || @@ -288,35 +305,11 @@ static void message_size_destroy_call_elem( calld->~call_data(); } -static int default_size(const grpc_channel_args* args, - int without_minimal_stack) { - if (grpc_channel_args_want_minimal_stack(args)) { - return -1; - } - return without_minimal_stack; -} - grpc_core::MessageSizeParsedConfig::message_size_limits get_message_size_limits( const grpc_channel_args* channel_args) { grpc_core::MessageSizeParsedConfig::message_size_limits lim; - lim.max_send_size = - default_size(channel_args, GRPC_DEFAULT_MAX_SEND_MESSAGE_LENGTH); - lim.max_recv_size = - default_size(channel_args, GRPC_DEFAULT_MAX_RECV_MESSAGE_LENGTH); - for (size_t i = 0; i < channel_args->num_args; ++i) { - if (strcmp(channel_args->args[i].key, GRPC_ARG_MAX_SEND_MESSAGE_LENGTH) == - 0) { - const grpc_integer_options options = {lim.max_send_size, -1, INT_MAX}; - lim.max_send_size = - grpc_channel_arg_get_integer(&channel_args->args[i], options); - } - if (strcmp(channel_args->args[i].key, - GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH) == 0) { - const grpc_integer_options options = {lim.max_recv_size, -1, INT_MAX}; - lim.max_recv_size = - grpc_channel_arg_get_integer(&channel_args->args[i], options); - } - } + lim.max_send_size = grpc_core::get_max_send_size(channel_args); + lim.max_recv_size = grpc_core::get_max_recv_size(channel_args); return lim; } @@ -327,26 +320,6 @@ static grpc_error* message_size_init_channel_elem( channel_data* chand = static_cast(elem->channel_data); new (chand) channel_data(); chand->limits = get_message_size_limits(args->channel_args); - // TODO(yashykt): We only need to read GRPC_ARG_SERVICE_CONFIG in the case of - // direct channels. (Service config is otherwise stored in the call_context by - // client_channel filter.) If we ever need a second filter that also needs to - // parse GRPC_ARG_SERVICE_CONFIG, we should refactor this code and add a - // separate filter that reads GRPC_ARG_SERVICE_CONFIG and saves the parsed - // config in the call_context. - const grpc_arg* channel_arg = - grpc_channel_args_find(args->channel_args, GRPC_ARG_SERVICE_CONFIG); - const char* service_config_str = grpc_channel_arg_get_string(channel_arg); - if (service_config_str != nullptr) { - grpc_error* service_config_error = GRPC_ERROR_NONE; - auto svc_cfg = grpc_core::ServiceConfig::Create(service_config_str, - &service_config_error); - if (service_config_error == GRPC_ERROR_NONE) { - chand->svc_cfg = std::move(svc_cfg); - } else { - gpr_log(GPR_ERROR, "%s", grpc_error_string(service_config_error)); - } - GRPC_ERROR_UNREF(service_config_error); - } return GRPC_ERROR_NONE; } @@ -387,6 +360,9 @@ static bool maybe_add_message_size_filter(grpc_channel_stack_builder* builder, void* /*arg*/) { const grpc_channel_args* channel_args = grpc_channel_stack_builder_get_channel_arguments(builder); + if (grpc_channel_args_want_minimal_stack(channel_args)) { + return true; + } bool enable = false; grpc_core::MessageSizeParsedConfig::message_size_limits lim = get_message_size_limits(channel_args); diff --git a/src/core/ext/filters/message_size/message_size_filter.h b/src/core/ext/filters/message_size/message_size_filter.h index 132d7b2af0f..ea0dd0266d0 100644 --- a/src/core/ext/filters/message_size/message_size_filter.h +++ b/src/core/ext/filters/message_size/message_size_filter.h @@ -40,6 +40,9 @@ class MessageSizeParsedConfig : public ServiceConfigParser::ParsedConfig { const message_size_limits& limits() const { return limits_; } + static const MessageSizeParsedConfig* GetFromCallContext( + const grpc_call_context_element* context); + private: message_size_limits limits_; }; @@ -54,6 +57,9 @@ class MessageSizeParser : public ServiceConfigParser::Parser { static size_t ParserIndex(); }; +int GetMaxRecvSizeFromChannelArgs(const grpc_channel_args* args); +int GetMaxSendSizeFromChannelArgs(const grpc_channel_args* args); + } // namespace grpc_core #endif /* GRPC_CORE_EXT_FILTERS_MESSAGE_SIZE_MESSAGE_SIZE_FILTER_H */ diff --git a/src/core/plugin_registry/grpc_plugin_registry.cc b/src/core/plugin_registry/grpc_plugin_registry.cc index f1c3cbf4036..ef8b10df647 100644 --- a/src/core/plugin_registry/grpc_plugin_registry.cc +++ b/src/core/plugin_registry/grpc_plugin_registry.cc @@ -64,6 +64,8 @@ void grpc_max_age_filter_init(void); void grpc_max_age_filter_shutdown(void); void grpc_message_size_filter_init(void); void grpc_message_size_filter_shutdown(void); +void grpc_service_config_channel_arg_filter_init(void); +void grpc_service_config_channel_arg_filter_shutdown(void); void grpc_client_authority_filter_init(void); void grpc_client_authority_filter_shutdown(void); void grpc_workaround_cronet_compression_filter_init(void); @@ -114,6 +116,8 @@ void grpc_register_built_in_plugins(void) { grpc_max_age_filter_shutdown); grpc_register_plugin(grpc_message_size_filter_init, grpc_message_size_filter_shutdown); + grpc_register_plugin(grpc_service_config_channel_arg_filter_init, + grpc_service_config_channel_arg_filter_shutdown); grpc_register_plugin(grpc_client_authority_filter_init, grpc_client_authority_filter_shutdown); grpc_register_plugin(grpc_workaround_cronet_compression_filter_init, diff --git a/src/core/plugin_registry/grpc_unsecure_plugin_registry.cc b/src/core/plugin_registry/grpc_unsecure_plugin_registry.cc index de53f173294..525fa108d81 100644 --- a/src/core/plugin_registry/grpc_unsecure_plugin_registry.cc +++ b/src/core/plugin_registry/grpc_unsecure_plugin_registry.cc @@ -64,6 +64,8 @@ void grpc_max_age_filter_init(void); void grpc_max_age_filter_shutdown(void); void grpc_message_size_filter_init(void); void grpc_message_size_filter_shutdown(void); +void grpc_service_config_channel_arg_filter_init(void); +void grpc_service_config_channel_arg_filter_shutdown(void); void grpc_client_authority_filter_init(void); void grpc_client_authority_filter_shutdown(void); void grpc_workaround_cronet_compression_filter_init(void); @@ -114,6 +116,8 @@ void grpc_register_built_in_plugins(void) { grpc_max_age_filter_shutdown); grpc_register_plugin(grpc_message_size_filter_init, grpc_message_size_filter_shutdown); + grpc_register_plugin(grpc_service_config_channel_arg_filter_init, + grpc_service_config_channel_arg_filter_shutdown); grpc_register_plugin(grpc_client_authority_filter_init, grpc_client_authority_filter_shutdown); grpc_register_plugin(grpc_workaround_cronet_compression_filter_init, diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 37bed16955f..7afe714e078 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -70,6 +70,7 @@ CORE_SOURCE_FILES = [ 'src/core/ext/filters/client_channel/retry_throttle.cc', 'src/core/ext/filters/client_channel/server_address.cc', 'src/core/ext/filters/client_channel/service_config.cc', + 'src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc', 'src/core/ext/filters/client_channel/service_config_parser.cc', 'src/core/ext/filters/client_channel/subchannel.cc', 'src/core/ext/filters/client_channel/subchannel_pool_interface.cc', diff --git a/test/core/end2end/tests/max_message_length.cc b/test/core/end2end/tests/max_message_length.cc index 40e752a3d63..256cc982940 100644 --- a/test/core/end2end/tests/max_message_length.cc +++ b/test/core/end2end/tests/max_message_length.cc @@ -29,6 +29,7 @@ #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/slice/slice_internal.h" +#include "src/core/lib/slice/slice_string_helpers.h" #include "src/core/lib/transport/metadata.h" #include "test/core/end2end/cq_verifier.h" @@ -466,6 +467,328 @@ static void test_max_message_length_on_response(grpc_end2end_test_config config, grpc_byte_buffer_destroy(response_payload); grpc_byte_buffer_destroy(recv_payload); + grpc_call_unref(c); + if (s != nullptr) grpc_call_unref(s); + cq_verifier_destroy(cqv); + end_test(&f); + config.tear_down_data(&f); +} + +static grpc_metadata gzip_compression_override() { + grpc_metadata gzip_compression_override; + gzip_compression_override.key = GRPC_MDSTR_GRPC_INTERNAL_ENCODING_REQUEST; + gzip_compression_override.value = grpc_slice_from_static_string("gzip"); + memset(&gzip_compression_override.internal_data, 0, + sizeof(gzip_compression_override.internal_data)); + return gzip_compression_override; +} + +// Test receive message limit with compressed request larger than the limit +static void test_max_receive_message_length_on_compressed_request( + grpc_end2end_test_config config, bool minimal_stack) { + gpr_log(GPR_INFO, + "test max receive message length on compressed request with " + "minimal_stack=%d", + minimal_stack); + grpc_end2end_test_fixture f; + grpc_call* c = nullptr; + grpc_call* s = nullptr; + cq_verifier* cqv; + grpc_op ops[6]; + grpc_op* op; + grpc_slice request_payload_slice = grpc_slice_malloc(1024); + memset(GRPC_SLICE_START_PTR(request_payload_slice), 'a', 1024); + grpc_byte_buffer* request_payload = + grpc_raw_byte_buffer_create(&request_payload_slice, 1); + grpc_byte_buffer* recv_payload = nullptr; + grpc_metadata_array initial_metadata_recv; + grpc_metadata_array trailing_metadata_recv; + grpc_metadata_array request_metadata_recv; + grpc_call_details call_details; + grpc_status_code status; + grpc_call_error error; + grpc_slice details, status_details; + int was_cancelled = 2; + + // Set limit via channel args. + grpc_arg arg[2]; + arg[0] = grpc_channel_arg_integer_create( + const_cast(GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH), 5); + arg[1] = grpc_channel_arg_integer_create( + const_cast(GRPC_ARG_MINIMAL_STACK), minimal_stack); + grpc_channel_args* server_args = + grpc_channel_args_copy_and_add(nullptr, arg, 2); + + f = begin_test(config, "test_max_request_message_length", nullptr, + server_args); + { + grpc_core::ExecCtx exec_ctx; + grpc_channel_args_destroy(server_args); + } + cqv = cq_verifier_create(f.cq); + c = grpc_channel_create_call(f.client, nullptr, GRPC_PROPAGATE_DEFAULTS, f.cq, + grpc_slice_from_static_string("/service/method"), + nullptr, gpr_inf_future(GPR_CLOCK_REALTIME), + nullptr); + GPR_ASSERT(c); + + grpc_metadata_array_init(&initial_metadata_recv); + grpc_metadata_array_init(&trailing_metadata_recv); + grpc_metadata_array_init(&request_metadata_recv); + grpc_call_details_init(&call_details); + + grpc_metadata compression_md = gzip_compression_override(); + memset(ops, 0, sizeof(ops)); + op = ops; + op->op = GRPC_OP_SEND_INITIAL_METADATA; + op->data.send_initial_metadata.count = 1; + op->data.send_initial_metadata.metadata = &compression_md; + op->flags = 0; + op->reserved = nullptr; + op++; + op->op = GRPC_OP_SEND_MESSAGE; + op->data.send_message.send_message = request_payload; + op->flags = 0; + op->reserved = nullptr; + op++; + op->op = GRPC_OP_SEND_CLOSE_FROM_CLIENT; + op->flags = 0; + op->reserved = nullptr; + op++; + op->op = GRPC_OP_RECV_INITIAL_METADATA; + op->data.recv_initial_metadata.recv_initial_metadata = &initial_metadata_recv; + op->flags = 0; + op->reserved = nullptr; + op++; + op->op = GRPC_OP_RECV_STATUS_ON_CLIENT; + op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv; + op->data.recv_status_on_client.status = &status; + op->data.recv_status_on_client.status_details = &details; + op->flags = 0; + op->reserved = nullptr; + op++; + error = grpc_call_start_batch(c, ops, static_cast(op - ops), tag(1), + nullptr); + GPR_ASSERT(GRPC_CALL_OK == error); + + error = + grpc_server_request_call(f.server, &s, &call_details, + &request_metadata_recv, f.cq, f.cq, tag(101)); + GPR_ASSERT(GRPC_CALL_OK == error); + CQ_EXPECT_COMPLETION(cqv, tag(101), 1); + cq_verify(cqv); + + memset(ops, 0, sizeof(ops)); + op = ops; + op->op = GRPC_OP_RECV_CLOSE_ON_SERVER; + op->data.recv_close_on_server.cancelled = &was_cancelled; + op->flags = 0; + op->reserved = nullptr; + op++; + op->op = GRPC_OP_RECV_MESSAGE; + op->data.recv_message.recv_message = &recv_payload; + op->flags = 0; + op->reserved = nullptr; + op++; + if (minimal_stack) { + /* Expect the RPC to proceed normally for a minimal stack */ + op->op = GRPC_OP_SEND_INITIAL_METADATA; + op->data.send_initial_metadata.count = 0; + op->flags = 0; + op->reserved = nullptr; + op++; + op->op = GRPC_OP_SEND_STATUS_FROM_SERVER; + op->data.send_status_from_server.trailing_metadata_count = 0; + op->data.send_status_from_server.status = GRPC_STATUS_OK; + status_details = grpc_slice_from_static_string("xyz"); + op->data.send_status_from_server.status_details = &status_details; + op->flags = 0; + op->reserved = nullptr; + op++; + } + error = grpc_call_start_batch(s, ops, static_cast(op - ops), tag(102), + nullptr); + GPR_ASSERT(GRPC_CALL_OK == error); + + CQ_EXPECT_COMPLETION(cqv, tag(102), 1); + CQ_EXPECT_COMPLETION(cqv, tag(1), 1); + cq_verify(cqv); + + GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/service/method")); + if (minimal_stack) { + /* We do not perform message size checks for minimal stack. */ + GPR_ASSERT(status == GRPC_STATUS_OK); + } else { + GPR_ASSERT(was_cancelled == 1); + GPR_ASSERT(status == GRPC_STATUS_RESOURCE_EXHAUSTED); + GPR_ASSERT(grpc_slice_str_cmp( + details, "Received message larger than max (29 vs. 5)") == + 0); + } + grpc_slice_unref(details); + grpc_slice_unref(request_payload_slice); + grpc_metadata_array_destroy(&initial_metadata_recv); + grpc_metadata_array_destroy(&trailing_metadata_recv); + grpc_metadata_array_destroy(&request_metadata_recv); + grpc_call_details_destroy(&call_details); + grpc_byte_buffer_destroy(request_payload); + grpc_byte_buffer_destroy(recv_payload); + grpc_call_unref(c); + if (s != nullptr) grpc_call_unref(s); + cq_verifier_destroy(cqv); + + end_test(&f); + config.tear_down_data(&f); +} + +// Test receive message limit with compressed response larger than the limit. +static void test_max_receive_message_length_on_compressed_response( + grpc_end2end_test_config config, bool minimal_stack) { + gpr_log(GPR_INFO, + "testing max receive message length on compressed response with " + "minimal_stack=%d", + minimal_stack); + grpc_end2end_test_fixture f; + grpc_call* c = nullptr; + grpc_call* s = nullptr; + cq_verifier* cqv; + grpc_op ops[6]; + grpc_op* op; + grpc_slice response_payload_slice = grpc_slice_malloc(1024); + memset(GRPC_SLICE_START_PTR(response_payload_slice), 'a', 1024); + grpc_byte_buffer* response_payload = + grpc_raw_byte_buffer_create(&response_payload_slice, 1); + grpc_byte_buffer* recv_payload = nullptr; + grpc_metadata_array initial_metadata_recv; + grpc_metadata_array trailing_metadata_recv; + grpc_metadata_array request_metadata_recv; + grpc_call_details call_details; + grpc_status_code status; + grpc_call_error error; + grpc_slice details; + int was_cancelled = 2; + + // Set limit via channel args. + grpc_arg arg[2]; + arg[0] = grpc_channel_arg_integer_create( + const_cast(GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH), 5); + arg[1] = grpc_channel_arg_integer_create( + const_cast(GRPC_ARG_MINIMAL_STACK), minimal_stack); + grpc_channel_args* client_args = + grpc_channel_args_copy_and_add(nullptr, arg, 2); + + f = begin_test(config, "test_max_response_message_length", client_args, + nullptr); + { + grpc_core::ExecCtx exec_ctx; + grpc_channel_args_destroy(client_args); + } + cqv = cq_verifier_create(f.cq); + + c = grpc_channel_create_call(f.client, nullptr, GRPC_PROPAGATE_DEFAULTS, f.cq, + grpc_slice_from_static_string("/service/method"), + nullptr, gpr_inf_future(GPR_CLOCK_REALTIME), + nullptr); + GPR_ASSERT(c); + + grpc_metadata_array_init(&initial_metadata_recv); + grpc_metadata_array_init(&trailing_metadata_recv); + grpc_metadata_array_init(&request_metadata_recv); + grpc_call_details_init(&call_details); + + memset(ops, 0, sizeof(ops)); + op = ops; + op->op = GRPC_OP_SEND_INITIAL_METADATA; + op->data.send_initial_metadata.count = 0; + op->flags = 0; + op->reserved = nullptr; + op++; + op->op = GRPC_OP_SEND_CLOSE_FROM_CLIENT; + op->flags = 0; + op->reserved = nullptr; + op++; + op->op = GRPC_OP_RECV_INITIAL_METADATA; + op->data.recv_initial_metadata.recv_initial_metadata = &initial_metadata_recv; + op->flags = 0; + op->reserved = nullptr; + op++; + op->op = GRPC_OP_RECV_MESSAGE; + op->data.recv_message.recv_message = &recv_payload; + op->flags = 0; + op->reserved = nullptr; + op++; + op->op = GRPC_OP_RECV_STATUS_ON_CLIENT; + op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv; + op->data.recv_status_on_client.status = &status; + op->data.recv_status_on_client.status_details = &details; + op->flags = 0; + op->reserved = nullptr; + op++; + error = grpc_call_start_batch(c, ops, static_cast(op - ops), tag(1), + nullptr); + GPR_ASSERT(GRPC_CALL_OK == error); + + error = + grpc_server_request_call(f.server, &s, &call_details, + &request_metadata_recv, f.cq, f.cq, tag(101)); + GPR_ASSERT(GRPC_CALL_OK == error); + CQ_EXPECT_COMPLETION(cqv, tag(101), 1); + cq_verify(cqv); + + grpc_metadata compression_md = gzip_compression_override(); + memset(ops, 0, sizeof(ops)); + op = ops; + op->op = GRPC_OP_SEND_INITIAL_METADATA; + op->data.send_initial_metadata.count = 1; + op->data.send_initial_metadata.metadata = &compression_md; + op->flags = 0; + op->reserved = nullptr; + op++; + op->op = GRPC_OP_RECV_CLOSE_ON_SERVER; + op->data.recv_close_on_server.cancelled = &was_cancelled; + op->flags = 0; + op->reserved = nullptr; + op++; + op->op = GRPC_OP_SEND_MESSAGE; + op->data.send_message.send_message = response_payload; + op->flags = 0; + op->reserved = nullptr; + op++; + op->op = GRPC_OP_SEND_STATUS_FROM_SERVER; + op->data.send_status_from_server.trailing_metadata_count = 0; + op->data.send_status_from_server.status = GRPC_STATUS_OK; + grpc_slice status_details = grpc_slice_from_static_string("xyz"); + op->data.send_status_from_server.status_details = &status_details; + op->flags = 0; + op->reserved = nullptr; + op++; + error = grpc_call_start_batch(s, ops, static_cast(op - ops), tag(102), + nullptr); + GPR_ASSERT(GRPC_CALL_OK == error); + + CQ_EXPECT_COMPLETION(cqv, tag(102), 1); + CQ_EXPECT_COMPLETION(cqv, tag(1), 1); + cq_verify(cqv); + + GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/service/method")); + if (minimal_stack) { + /* We do not perform message size checks for minimal stack. */ + GPR_ASSERT(status == GRPC_STATUS_OK); + } else { + GPR_ASSERT(status == GRPC_STATUS_RESOURCE_EXHAUSTED); + GPR_ASSERT(grpc_slice_str_cmp( + details, "Received message larger than max (29 vs. 5)") == + 0); + } + grpc_slice_unref(details); + grpc_slice_unref(response_payload_slice); + grpc_metadata_array_destroy(&initial_metadata_recv); + grpc_metadata_array_destroy(&trailing_metadata_recv); + grpc_metadata_array_destroy(&request_metadata_recv); + grpc_call_details_destroy(&call_details); + grpc_byte_buffer_destroy(response_payload); + grpc_byte_buffer_destroy(recv_payload); + grpc_call_unref(c); if (s != nullptr) grpc_call_unref(s); @@ -500,6 +823,15 @@ void max_message_length(grpc_end2end_test_config config) { test_max_message_length_on_response(config, false /* send_limit */, true /* use_service_config */, true /* use_string_json_value */); + /* The following tests are not useful for inproc transport and do not work + * with our simple proxy. */ + if (strcmp(config.name, "inproc") != 0 && + (config.feature_mask & FEATURE_MASK_SUPPORTS_REQUEST_PROXYING) == 0) { + test_max_receive_message_length_on_compressed_request(config, false); + test_max_receive_message_length_on_compressed_request(config, true); + test_max_receive_message_length_on_compressed_response(config, false); + test_max_receive_message_length_on_compressed_response(config, true); + } } void max_message_length_pre_init(void) {} diff --git a/test/cpp/microbenchmarks/bm_call_create.cc b/test/cpp/microbenchmarks/bm_call_create.cc index 803ad38f1c5..b4a70272ea6 100644 --- a/test/cpp/microbenchmarks/bm_call_create.cc +++ b/test/cpp/microbenchmarks/bm_call_create.cc @@ -529,9 +529,10 @@ static void BM_IsolatedFilter(benchmark::State& state) { grpc_call_final_info final_info; TestOp test_op_data; const int kArenaSize = 4096; + grpc_call_context_element context[GRPC_CONTEXT_COUNT] = {}; grpc_call_element_args call_args{call_stack, nullptr, - nullptr, + context, method, start_time, deadline, diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 613e824f4d7..3a4fdf1b017 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -1162,6 +1162,7 @@ src/core/ext/filters/client_channel/server_address.h \ src/core/ext/filters/client_channel/service_config.cc \ src/core/ext/filters/client_channel/service_config.h \ src/core/ext/filters/client_channel/service_config_call_data.h \ +src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc \ src/core/ext/filters/client_channel/service_config_parser.cc \ src/core/ext/filters/client_channel/service_config_parser.h \ src/core/ext/filters/client_channel/subchannel.cc \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index b026c8b1f8a..1ed428487fe 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -962,6 +962,7 @@ src/core/ext/filters/client_channel/server_address.h \ src/core/ext/filters/client_channel/service_config.cc \ src/core/ext/filters/client_channel/service_config.h \ src/core/ext/filters/client_channel/service_config_call_data.h \ +src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc \ src/core/ext/filters/client_channel/service_config_parser.cc \ src/core/ext/filters/client_channel/service_config_parser.h \ src/core/ext/filters/client_channel/subchannel.cc \ From 5041fcc1caca525067e2f02d585effda00681d1f Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Fri, 19 Jun 2020 11:46:16 -0700 Subject: [PATCH 78/80] Reviewer comments --- .../service_config_channel_arg_filter.cc | 71 ++++++++++--------- .../message_decompress_filter.cc | 4 +- .../message_size/message_size_filter.cc | 6 +- 3 files changed, 41 insertions(+), 40 deletions(-) diff --git a/src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc b/src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc index 9aad7c71307..9c7af64bd97 100644 --- a/src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc +++ b/src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc @@ -29,20 +29,18 @@ namespace grpc_core { namespace { -class ServiceConfigChannelArgCallData {}; - class ServiceConfigChannelArgChannelData { public: explicit ServiceConfigChannelArgChannelData( const grpc_channel_element_args* args) { - const char* service_config_str = grpc_channel_arg_find_string( + const char* service_config_str = grpc_channel_args_find_string( args->channel_args, GRPC_ARG_SERVICE_CONFIG); if (service_config_str != nullptr) { grpc_error* service_config_error = GRPC_ERROR_NONE; - auto svc_cfg = grpc_core::ServiceConfig::Create(service_config_str, - &service_config_error); + auto service_config = grpc_core::ServiceConfig::Create( + service_config_str, &service_config_error); if (service_config_error == GRPC_ERROR_NONE) { - svc_cfg_ = std::move(svc_cfg); + service_config_ = std::move(service_config); } else { gpr_log(GPR_ERROR, "%s", grpc_error_string(service_config_error)); } @@ -50,50 +48,53 @@ class ServiceConfigChannelArgChannelData { } } - grpc_core::RefCountedPtr svc_cfg() const { - return svc_cfg_; + grpc_core::RefCountedPtr service_config() const { + return service_config_; } private: - grpc_core::RefCountedPtr svc_cfg_; + grpc_core::RefCountedPtr service_config_; +}; + +class ServiceConfigChannelArgCallData { + public: + ServiceConfigChannelArgCallData(grpc_call_element* elem, + const grpc_call_element_args* args) { + ServiceConfigChannelArgChannelData* chand = + static_cast(elem->channel_data); + grpc_core::RefCountedPtr service_config = + chand->service_config(); + if (service_config != nullptr) { + GPR_DEBUG_ASSERT(args->context != nullptr); + const auto* method_params_vector = + service_config->GetMethodParsedConfigVector(args->path); + args->arena->New( + std::move(service_config), method_params_vector, args->context); + } + } }; grpc_error* ServiceConfigChannelArgInitCallElem( grpc_call_element* elem, const grpc_call_element_args* args) { - ServiceConfigChannelArgChannelData* chand = - static_cast(elem->channel_data); - if (chand->svc_cfg() != nullptr) { - GPR_DEBUG_ASSERT(args->context != nullptr); - args->arena->New( - chand->svc_cfg(), - chand->svc_cfg()->GetMethodParsedConfigVector(args->path), - args->context); - } + ServiceConfigChannelArgCallData* calld = + static_cast(elem->call_data); + new (calld) ServiceConfigChannelArgCallData(elem, args); return GRPC_ERROR_NONE; } void ServiceConfigChannelArgDestroyCallElem( - grpc_call_element* /* elem */, const grpc_call_final_info* /* final_info */, - grpc_closure* /* then_schedule_closure */) {} + grpc_call_element* elem, const grpc_call_final_info* /* final_info */, + grpc_closure* /* then_schedule_closure */) { + ServiceConfigChannelArgCallData* calld = + static_cast(elem->call_data); + calld->~ServiceConfigChannelArgCallData(); +} grpc_error* ServiceConfigChannelArgInitChannelElem( grpc_channel_element* elem, grpc_channel_element_args* args) { ServiceConfigChannelArgChannelData* chand = static_cast(elem->channel_data); new (chand) ServiceConfigChannelArgChannelData(args); - const char* service_config_str = grpc_channel_arg_get_string( - grpc_channel_args_find(args->channel_args, GRPC_ARG_SERVICE_CONFIG)); - if (service_config_str != nullptr) { - grpc_error* service_config_error = GRPC_ERROR_NONE; - auto svc_cfg = grpc_core::ServiceConfig::Create(service_config_str, - &service_config_error); - if (service_config_error == GRPC_ERROR_NONE) { - chand->svc_cfg = std::move(svc_cfg); - } else { - gpr_log(GPR_ERROR, "%s", grpc_error_string(service_config_error)); - } - GRPC_ERROR_UNREF(service_config_error); - } return GRPC_ERROR_NONE; } @@ -121,8 +122,8 @@ bool maybe_add_service_config_channel_arg_filter( const grpc_channel_args* channel_args = grpc_channel_stack_builder_get_channel_arguments(builder); if (grpc_channel_args_want_minimal_stack(channel_args) || - grpc_channel_arg_get_string(grpc_channel_args_find( - channel_args, GRPC_ARG_SERVICE_CONFIG)) == nullptr) { + grpc_channel_args_find_string(channel_args, GRPC_ARG_SERVICE_CONFIG) == + nullptr) { return true; } return grpc_channel_stack_builder_prepend_filter( diff --git a/src/core/ext/filters/http/message_compress/message_decompress_filter.cc b/src/core/ext/filters/http/message_compress/message_decompress_filter.cc index 122c74e1e98..c16ea66e7a3 100644 --- a/src/core/ext/filters/http/message_compress/message_decompress_filter.cc +++ b/src/core/ext/filters/http/message_compress/message_decompress_filter.cc @@ -46,7 +46,7 @@ namespace { class ChannelData { public: explicit ChannelData(const grpc_channel_element_args* args) - : max_recv_size_(get_max_recv_size(args->channel_args)) {} + : max_recv_size_(GetMaxRecvSizeFromChannelArgs(args->channel_args)) {} int max_recv_size() const { return max_recv_size_; } @@ -74,7 +74,7 @@ class CallData { OnRecvTrailingMetadataReady, this, grpc_schedule_on_exec_ctx); const MessageSizeParsedConfig* limits = - get_message_size_config_from_call_context(args.context); + MessageSizeParsedConfig::GetFromCallContext(args.context); if (limits != nullptr && limits->limits().max_recv_size >= 0 && (limits->limits().max_recv_size < max_recv_message_length_ || max_recv_message_length_ < 0)) { diff --git a/src/core/ext/filters/message_size/message_size_filter.cc b/src/core/ext/filters/message_size/message_size_filter.cc index 53b4da451f5..89fdab6fae8 100644 --- a/src/core/ext/filters/message_size/message_size_filter.cc +++ b/src/core/ext/filters/message_size/message_size_filter.cc @@ -152,7 +152,7 @@ struct call_data { // apply the max request size to the send limit and the max response // size to the receive limit. const grpc_core::MessageSizeParsedConfig* limits = - grpc_core::get_message_size_config_from_call_context(args.context); + grpc_core::MessageSizeParsedConfig::GetFromCallContext(args.context); if (limits != nullptr) { if (limits->limits().max_send_size >= 0 && (limits->limits().max_send_size < this->limits.max_send_size || @@ -308,8 +308,8 @@ static void message_size_destroy_call_elem( grpc_core::MessageSizeParsedConfig::message_size_limits get_message_size_limits( const grpc_channel_args* channel_args) { grpc_core::MessageSizeParsedConfig::message_size_limits lim; - lim.max_send_size = grpc_core::get_max_send_size(channel_args); - lim.max_recv_size = grpc_core::get_max_recv_size(channel_args); + lim.max_send_size = grpc_core::GetMaxSendSizeFromChannelArgs(channel_args); + lim.max_recv_size = grpc_core::GetMaxRecvSizeFromChannelArgs(channel_args); return lim; } From b57a11a82bf803f56893881f3c0a0ac19abaf27b Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Fri, 19 Jun 2020 14:47:41 -0700 Subject: [PATCH 79/80] Reviewer comments --- .../service_config_channel_arg_filter.cc | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc b/src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc index 9c7af64bd97..6b8e6373af7 100644 --- a/src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc +++ b/src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc @@ -37,8 +37,8 @@ class ServiceConfigChannelArgChannelData { args->channel_args, GRPC_ARG_SERVICE_CONFIG); if (service_config_str != nullptr) { grpc_error* service_config_error = GRPC_ERROR_NONE; - auto service_config = grpc_core::ServiceConfig::Create( - service_config_str, &service_config_error); + auto service_config = + ServiceConfig::Create(service_config_str, &service_config_error); if (service_config_error == GRPC_ERROR_NONE) { service_config_ = std::move(service_config); } else { @@ -48,12 +48,12 @@ class ServiceConfigChannelArgChannelData { } } - grpc_core::RefCountedPtr service_config() const { + RefCountedPtr service_config() const { return service_config_; } private: - grpc_core::RefCountedPtr service_config_; + RefCountedPtr service_config_; }; class ServiceConfigChannelArgCallData { @@ -62,8 +62,7 @@ class ServiceConfigChannelArgCallData { const grpc_call_element_args* args) { ServiceConfigChannelArgChannelData* chand = static_cast(elem->channel_data); - grpc_core::RefCountedPtr service_config = - chand->service_config(); + RefCountedPtr service_config = chand->service_config(); if (service_config != nullptr) { GPR_DEBUG_ASSERT(args->context != nullptr); const auto* method_params_vector = From 2a3b1f0eb7b8424fef4a3636d99abb93bd0d7ca2 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Mon, 22 Jun 2020 09:25:59 -0700 Subject: [PATCH 80/80] Revert log changes --- test/cpp/qps/qps_worker.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/cpp/qps/qps_worker.cc b/test/cpp/qps/qps_worker.cc index 5d6a2fa1f1a..c364216bc02 100644 --- a/test/cpp/qps/qps_worker.cc +++ b/test/cpp/qps/qps_worker.cc @@ -292,10 +292,11 @@ QpsWorker::QpsWorker(int driver_port, int server_port, server_ = builder->BuildAndStart(); if (server_ == nullptr) { gpr_log(GPR_ERROR, - "QpsWorker: Fail to BuildAndStart(port=%d) (server_port=%d)", + "QpsWorker: Fail to BuildAndStart(driver_port=%d, server_port=%d)", driver_port, server_port); } else { - gpr_log(GPR_INFO, "QpsWorker: BuildAndStart(port=%d) (server_port=%d)", + gpr_log(GPR_INFO, + "QpsWorker: BuildAndStart(driver_port=%d, server_port=%d) done", driver_port, server_port); } }