From 02135239077400767453808989ee1d21e1332692 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Mon, 4 Mar 2024 11:04:24 -0800 Subject: [PATCH 01/52] [build] move channel out of grpc_base (#35924) This adds the following new targets: - `channel`: A virtual interface for a channel. - `legacy_channel`: A channel implementation that supports the filter stack and call v2. - `channel_create`: A standalone function to create a channel. - `server_interface`: A base class with a few accessor methods used in surface/call.cc. - `server`: The actual server implementation. - `api_trace`, `call_tracer`, `server_call_tracer_filter`, `call_finalization`: These were split out of `grpc_base` to avoid various dependency problems. - `compression`: This is a combination of the previously existing `compression_internal` target and the compression code that was part of `grpc_base`. Closes #35924 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/35924 from markdroth:channel_interface 94a7fffddb644375cd49ae2c7aec142548db0d2b PiperOrigin-RevId: 612512438 --- BUILD | 295 +++++++++++- CMakeLists.txt | 16 +- Makefile | 10 +- Package.swift | 9 +- build_autogenerated.yaml | 28 +- config.m4 | 5 +- config.w32 | 5 +- gRPC-C++.podspec | 8 +- gRPC-Core.podspec | 13 +- grpc.gemspec | 9 +- grpc.gyp | 14 +- package.xml | 9 +- src/core/BUILD | 98 +++- .../client_channel/channel_connectivity.cc | 250 ---------- .../client_channel/client_channel_filter.cc | 11 - .../client_channel/client_channel_filter.h | 5 - .../binder/client/channel_create_impl.cc | 9 +- .../client/chaotic_good_connector.cc | 3 +- .../chttp2/client/chttp2_connector.cc | 10 +- src/core/ext/transport/cronet/BUILD | 2 + .../client/secure/cronet_channel_create.cc | 5 +- .../ext/transport/inproc/inproc_transport.cc | 9 +- .../inproc/legacy_inproc_transport.cc | 3 +- src/core/ext/xds/xds_transport_grpc.cc | 47 +- src/core/ext/xds/xds_transport_grpc.h | 6 +- src/core/lib/channel/call_finalization.h | 2 +- src/core/lib/channel/call_tracer.h | 2 +- src/core/lib/channel/channelz.h | 3 + .../lib/compression/compression_internal.cc | 33 ++ .../lib/compression/compression_internal.h | 3 + src/core/lib/surface/builtins.h | 26 -- src/core/lib/surface/call.cc | 16 +- src/core/lib/surface/call.h | 6 +- src/core/lib/surface/channel.cc | 426 +++++------------- src/core/lib/surface/channel.h | 195 ++++---- src/core/lib/surface/channel_create.cc | 100 ++++ .../surface/{builtins.cc => channel_create.h} | 35 +- src/core/lib/surface/channel_ping.cc | 69 --- src/core/lib/surface/lame_client.cc | 31 +- src/core/lib/surface/lame_client.h | 9 +- src/core/lib/surface/legacy_channel.cc | 418 +++++++++++++++++ src/core/lib/surface/legacy_channel.h | 118 +++++ src/core/lib/surface/server.cc | 19 +- src/core/lib/surface/server.h | 16 +- src/core/lib/surface/server_interface.h | 43 ++ src/core/load_balancing/grpclb/grpclb.cc | 43 +- src/core/load_balancing/rls/rls.cc | 42 +- .../plugin_registry/grpc_plugin_registry.cc | 19 +- .../grpc_plugin_registry_extra.cc | 1 - .../grpc_plugin_registry_noextra.cc | 1 - .../alts/handshaker/alts_handshaker_client.cc | 10 +- .../alts/handshaker/alts_tsi_handshaker.cc | 1 + src/cpp/ext/csm/BUILD | 1 + src/cpp/ext/otel/BUILD | 1 + src/python/grpcio/grpc_core_dependencies.py | 5 +- test/core/bad_connection/BUILD | 1 + test/core/bad_connection/close_fd_test.cc | 4 +- test/core/end2end/fixtures/sockpair_fixture.h | 5 +- test/core/end2end/fuzzers/client_fuzzer.cc | 5 +- test/core/surface/lame_client_test.cc | 4 +- .../surface/secure_channel_create_test.cc | 13 +- .../binder/end2end/fuzzers/client_fuzzer.cc | 5 +- .../binder/end2end/testing_channel_create.cc | 3 +- test/core/transport/chaotic_good/BUILD | 5 +- test/cpp/microbenchmarks/fullstack_fixtures.h | 5 +- test/cpp/performance/BUILD | 1 + test/cpp/performance/writes_per_rpc_test.cc | 5 +- tools/doxygen/Doxyfile.c++.internal | 9 +- tools/doxygen/Doxyfile.core.internal | 9 +- .../run_tests/sanity/core_banned_functions.py | 1 + 70 files changed, 1559 insertions(+), 1089 deletions(-) delete mode 100644 src/core/client_channel/channel_connectivity.cc delete mode 100644 src/core/lib/surface/builtins.h create mode 100644 src/core/lib/surface/channel_create.cc rename src/core/lib/surface/{builtins.cc => channel_create.h} (53%) delete mode 100644 src/core/lib/surface/channel_ping.cc create mode 100644 src/core/lib/surface/legacy_channel.cc create mode 100644 src/core/lib/surface/legacy_channel.h create mode 100644 src/core/lib/surface/server_interface.h diff --git a/BUILD b/BUILD index dd7046fbc4e..617bc1cf533 100644 --- a/BUILD +++ b/BUILD @@ -554,6 +554,7 @@ grpc_cc_library( ], visibility = ["@grpc:public"], deps = [ + "api_trace", "channel_arg_names", "channel_stack_builder", "config", @@ -567,6 +568,7 @@ grpc_cc_library( "grpc_trace", "http_connect_handshaker", "iomgr_timer", + "server", "//src/core:channel_args", "//src/core:channel_init", "//src/core:channel_stack_type", @@ -578,6 +580,7 @@ grpc_cc_library( "//src/core:http_proxy_mapper", "//src/core:init_internally", "//src/core:posix_event_engine_timer_manager", + "//src/core:server_call_tracer_filter", "//src/core:service_config_channel_arg_filter", "//src/core:slice", "//src/core:tcp_connect_handshaker", @@ -629,6 +632,7 @@ grpc_cc_library( "@grpc:public", ], deps = [ + "api_trace", "channel_arg_names", "channel_stack_builder", "config", @@ -649,6 +653,7 @@ grpc_cc_library( "iomgr_timer", "promise", "ref_counted_ptr", + "server", "sockaddr_utils", "tsi_base", "uri_parser", @@ -676,6 +681,7 @@ grpc_cc_library( "//src/core:json", "//src/core:posix_event_engine_timer_manager", "//src/core:ref_counted", + "//src/core:server_call_tracer_filter", "//src/core:service_config_channel_arg_filter", "//src/core:slice", "//src/core:slice_refcount", @@ -1101,6 +1107,9 @@ grpc_cc_library( ], tags = ["nofixdeps"], deps = [ + "api_trace", + "channel", + "channel_create", "config", "debug_location", "exec_ctx", @@ -1113,6 +1122,7 @@ grpc_cc_library( "grpc_public_hdrs", "orphanable", "ref_counted_ptr", + "server", "//src/core:arena", "//src/core:channel_args", "//src/core:channel_args_preconditioning", @@ -1251,6 +1261,7 @@ grpc_cc_library( ], visibility = ["@grpc:public"], deps = [ + "api_trace", "gpr", "grpc_base", "grpc_public_hdrs", @@ -1405,6 +1416,21 @@ grpc_cc_library( ], ) +grpc_cc_library( + name = "api_trace", + srcs = [ + "//src/core:lib/surface/api_trace.cc", + ], + hdrs = [ + "//src/core:lib/surface/api_trace.h", + ], + language = "c++", + deps = [ + "gpr", + "grpc_trace", + ], +) + grpc_cc_library( name = "byte_buffer", srcs = [ @@ -1416,6 +1442,7 @@ grpc_cc_library( "exec_ctx", "gpr_public_hdrs", "grpc_public_hdrs", + "//src/core:compression", "//src/core:slice", ], ) @@ -1626,31 +1653,253 @@ grpc_cc_library( ) grpc_cc_library( - name = "grpc_base", + name = "call_tracer", srcs = [ "//src/core:lib/channel/call_tracer.cc", + ], + hdrs = [ + "//src/core:lib/channel/call_tracer.h", + ], + external_deps = [ + "absl/status", + "absl/strings", + ], + language = "c++", + visibility = ["@grpc:alt_grpc_base_legacy"], + deps = [ + "config", + "gpr", + "legacy_context", + "tcp_tracer", + "//src/core:arena", + "//src/core:call_final_info", + "//src/core:channel_args", + "//src/core:context", + "//src/core:error", + "//src/core:metadata_batch", + "//src/core:slice_buffer", + ], +) + +grpc_cc_library( + name = "channel", + srcs = [ + "//src/core:lib/surface/channel.cc", + ], + hdrs = [ + "//src/core:lib/surface/channel.h", + ], + external_deps = [ + "absl/base:core_headers", + "absl/status:statusor", + "absl/strings", + "absl/types:optional", + ], + language = "c++", + visibility = ["@grpc:alt_grpc_base_legacy"], + deps = [ + "api_trace", + "channel_arg_names", + "channelz", + "cpp_impl_of", + "event_engine_base_hdrs", + "exec_ctx", + "gpr", + "grpc_public_hdrs", + "grpc_trace", + "orphanable", + "ref_counted_ptr", + "stats", + "//src/core:arena", + "//src/core:channel_args", + "//src/core:channel_stack_type", + "//src/core:compression", + "//src/core:connectivity_state", + "//src/core:iomgr_fwd", + "//src/core:ref_counted", + "//src/core:slice", + "//src/core:stats_data", + "//src/core:time", + ], +) + +grpc_cc_library( + name = "legacy_channel", + srcs = [ + "//src/core:lib/surface/legacy_channel.cc", + ], + hdrs = [ + "//src/core:lib/surface/legacy_channel.h", + ], + external_deps = [ + "absl/base:core_headers", + "absl/status", + "absl/status:statusor", + "absl/types:optional", + ], + language = "c++", + visibility = ["@grpc:alt_grpc_base_legacy"], + deps = [ + "api_trace", + "channel", + "channelz", + "config", + "exec_ctx", + "gpr", + "grpc_base", + "grpc_client_channel", + "ref_counted_ptr", + "stats", + "//src/core:arena", + "//src/core:call_factory", + "//src/core:channel_args", + "//src/core:channel_fwd", + "//src/core:channel_init", + "//src/core:channel_stack_type", + "//src/core:closure", + "//src/core:dual_ref_counted", + "//src/core:error", + "//src/core:init_internally", + "//src/core:iomgr_fwd", + "//src/core:metrics", + "//src/core:slice", + "//src/core:stats_data", + "//src/core:time", + ], +) + +grpc_cc_library( + name = "channel_create", + srcs = [ + "//src/core:lib/surface/channel_create.cc", + ], + hdrs = [ + "//src/core:lib/surface/channel_create.h", + ], + external_deps = [ + "absl/base:core_headers", + "absl/status:statusor", + "absl/strings", + "absl/types:optional", + ], + language = "c++", + visibility = ["@grpc:alt_grpc_base_legacy"], + deps = [ + "channel", + "channel_arg_names", + "channelz", + "gpr", + "grpc_base", + "grpc_public_hdrs", + "legacy_channel", + "ref_counted_ptr", + "stats", + "//src/core:arena", + "//src/core:channel_args", + "//src/core:channel_stack_type", + "//src/core:iomgr_fwd", + "//src/core:ref_counted", + "//src/core:slice", + "//src/core:stats_data", + ], +) + +grpc_cc_library( + name = "server", + srcs = [ + "//src/core:lib/surface/server.cc", + ], + hdrs = [ + "//src/core:lib/surface/server.h", + ], + external_deps = [ + "absl/base:core_headers", + "absl/cleanup", + "absl/container:flat_hash_map", + "absl/container:flat_hash_set", + "absl/hash", + "absl/random", + "absl/status", + "absl/status:statusor", + "absl/strings", + "absl/types:optional", + ], + language = "c++", + visibility = ["@grpc:alt_grpc_base_legacy"], + deps = [ + "api_trace", + "call_combiner", + "call_tracer", + "channel", + "channel_arg_names", + "channelz", + "config", + "cpp_impl_of", + "debug_location", + "exec_ctx", + "gpr", + "grpc_base", + "grpc_public_hdrs", + "grpc_trace", + "iomgr", + "legacy_channel", + "orphanable", + "promise", + "ref_counted_ptr", + "stats", + "//src/core:1999", + "//src/core:activity", + "//src/core:arena", + "//src/core:arena_promise", + "//src/core:cancel_callback", + "//src/core:channel_args", + "//src/core:channel_args_preconditioning", + "//src/core:channel_fwd", + "//src/core:channel_stack_type", + "//src/core:closure", + "//src/core:connectivity_state", + "//src/core:context", + "//src/core:dual_ref_counted", + "//src/core:error", + "//src/core:error_utils", + "//src/core:experiments", + "//src/core:iomgr_fwd", + "//src/core:map", + "//src/core:metadata_batch", + "//src/core:pipe", + "//src/core:poll", + "//src/core:pollset_set", + "//src/core:random_early_detection", + "//src/core:seq", + "//src/core:server_interface", + "//src/core:slice", + "//src/core:slice_buffer", + "//src/core:status_helper", + "//src/core:time", + "//src/core:try_join", + "//src/core:try_seq", + "//src/core:useful", + ], +) + +grpc_cc_library( + name = "grpc_base", + srcs = [ "//src/core:lib/channel/channel_stack.cc", "//src/core:lib/channel/channel_stack_builder_impl.cc", "//src/core:lib/channel/connected_channel.cc", "//src/core:lib/channel/promise_based_filter.cc", - "//src/core:lib/channel/server_call_tracer_filter.cc", "//src/core:lib/channel/status_util.cc", - "//src/core:lib/compression/compression.cc", "//src/core:lib/compression/message_compress.cc", "//src/core:lib/slice/b64.cc", - "//src/core:lib/surface/api_trace.cc", - "//src/core:lib/surface/builtins.cc", "//src/core:lib/surface/call.cc", "//src/core:lib/surface/call_details.cc", "//src/core:lib/surface/call_log_batch.cc", - "//src/core:lib/surface/channel.cc", - "//src/core:lib/surface/channel_ping.cc", "//src/core:lib/surface/completion_queue.cc", "//src/core:lib/surface/completion_queue_factory.cc", "//src/core:lib/surface/event_string.cc", "//src/core:lib/surface/lame_client.cc", "//src/core:lib/surface/metadata_array.cc", - "//src/core:lib/surface/server.cc", "//src/core:lib/surface/validate_metadata.cc", "//src/core:lib/surface/version.cc", "//src/core:lib/surface/wait_for_cq_end_op.cc", @@ -1659,8 +1908,6 @@ grpc_cc_library( "//src/core:lib/transport/transport_op_string.cc", ], hdrs = [ - "//src/core:lib/channel/call_finalization.h", - "//src/core:lib/channel/call_tracer.h", "//src/core:lib/channel/channel_stack.h", "//src/core:lib/channel/channel_stack_builder_impl.h", "//src/core:lib/channel/connected_channel.h", @@ -1668,17 +1915,13 @@ grpc_cc_library( "//src/core:lib/channel/status_util.h", "//src/core:lib/compression/message_compress.h", "//src/core:lib/slice/b64.h", - "//src/core:lib/surface/api_trace.h", - "//src/core:lib/surface/builtins.h", "//src/core:lib/surface/call.h", "//src/core:lib/surface/call_test_only.h", - "//src/core:lib/surface/channel.h", "//src/core:lib/surface/completion_queue.h", "//src/core:lib/surface/completion_queue_factory.h", "//src/core:lib/surface/event_string.h", "//src/core:lib/surface/init.h", "//src/core:lib/surface/lame_client.h", - "//src/core:lib/surface/server.h", "//src/core:lib/surface/validate_metadata.h", "//src/core:lib/surface/wait_for_cq_end_op.h", "//src/core:lib/transport/batch_builder.h", @@ -1716,9 +1959,12 @@ grpc_cc_library( public_hdrs = GRPC_PUBLIC_HDRS + GRPC_PUBLIC_EVENT_ENGINE_HDRS, visibility = ["@grpc:alt_grpc_base_legacy"], deps = [ + "api_trace", "byte_buffer", "call_combiner", "call_trace", + "call_tracer", + "channel", "channel_arg_names", "channel_stack_builder", "channelz", @@ -1755,6 +2001,7 @@ grpc_cc_library( "//src/core:call_factory", "//src/core:call_filters", "//src/core:call_final_info", + "//src/core:call_finalization", "//src/core:call_spine", "//src/core:cancel_callback", "//src/core:channel_args", @@ -1765,7 +2012,7 @@ grpc_cc_library( "//src/core:channel_stack_trace", "//src/core:channel_stack_type", "//src/core:closure", - "//src/core:compression_internal", + "//src/core:compression", "//src/core:connectivity_state", "//src/core:construct_destruct", "//src/core:context", @@ -1818,6 +2065,7 @@ grpc_cc_library( "//src/core:resource_quota", "//src/core:resource_quota_trace", "//src/core:seq", + "//src/core:server_interface", "//src/core:slice", "//src/core:slice_buffer", "//src/core:slice_cast", @@ -2013,6 +2261,7 @@ grpc_cc_library( public_hdrs = GRPC_PUBLIC_HDRS, visibility = ["@grpc:public"], deps = [ + "api_trace", "call_trace", "channel_arg_names", "channelz", @@ -2196,6 +2445,7 @@ grpc_cc_library( "legacy_context", "ref_counted_ptr", "resource_quota_api", + "server", "//src/core:arena", "//src/core:channel_args", "//src/core:channel_fwd", @@ -2274,6 +2524,7 @@ grpc_cc_library( "legacy_context", "ref_counted_ptr", "resource_quota_api", + "server", "//src/core:arena", "//src/core:channel_args", "//src/core:channel_init", @@ -2539,6 +2790,7 @@ grpc_cc_library( ], visibility = ["@grpc:grpc++_test"], deps = [ + "channel", "grpc++", "grpc_base", ], @@ -2585,6 +2837,7 @@ grpc_cc_library( language = "c++", visibility = ["@grpc:grpc_opencensus_plugin"], deps = [ + "call_tracer", "config", "gpr", "grpc++_base", @@ -3313,7 +3566,6 @@ grpc_cc_library( grpc_cc_library( name = "grpc_client_channel", srcs = [ - "//src/core:client_channel/channel_connectivity.cc", "//src/core:client_channel/client_channel_factory.cc", "//src/core:client_channel/client_channel_filter.cc", "//src/core:client_channel/client_channel_plugin.cc", @@ -3356,8 +3608,10 @@ grpc_cc_library( language = "c++", visibility = ["@grpc:client_channel"], deps = [ + "api_trace", "backoff", "call_combiner", + "call_tracer", "channel_arg_names", "channelz", "config", @@ -3670,6 +3924,7 @@ grpc_cc_library( language = "c++", visibility = ["@grpc:public"], deps = [ + "api_trace", "exec_ctx", "gpr", "grpc_base", @@ -3757,6 +4012,9 @@ grpc_cc_library( deps = [ "alts_upb", "alts_util", + "channel", + "channel_create", + "exec_ctx", "gpr", "grpc_base", "grpc_security_base", @@ -3921,6 +4179,7 @@ grpc_cc_library( visibility = ["@grpc:http"], deps = [ "call_trace", + "call_tracer", "channel_arg_names", "config", "gpr", @@ -3935,7 +4194,7 @@ grpc_cc_library( "//src/core:channel_args", "//src/core:channel_fwd", "//src/core:channel_stack_type", - "//src/core:compression_internal", + "//src/core:compression", "//src/core:context", "//src/core:experiments", "//src/core:grpc_message_size_filter", @@ -4217,6 +4476,7 @@ grpc_cc_library( "absl/types:variant", ], deps = [ + "call_tracer", "chttp2_legacy_frame", "gpr", "gpr_platform", @@ -4367,6 +4627,7 @@ grpc_cc_library( language = "c++", visibility = ["@grpc:grpclb"], deps = [ + "call_tracer", "channel_arg_names", "channelz", "chttp2_context_list_entry", diff --git a/CMakeLists.txt b/CMakeLists.txt index 31baf6469d0..d37c313418e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1778,7 +1778,6 @@ endif() add_library(grpc src/core/client_channel/backup_poller.cc - src/core/client_channel/channel_connectivity.cc src/core/client_channel/client_channel_channelz.cc src/core/client_channel/client_channel_factory.cc src/core/client_channel/client_channel_filter.cc @@ -2453,15 +2452,14 @@ add_library(grpc src/core/lib/slice/slice_refcount.cc src/core/lib/slice/slice_string_helpers.cc src/core/lib/surface/api_trace.cc - src/core/lib/surface/builtins.cc src/core/lib/surface/byte_buffer.cc src/core/lib/surface/byte_buffer_reader.cc src/core/lib/surface/call.cc src/core/lib/surface/call_details.cc src/core/lib/surface/call_log_batch.cc src/core/lib/surface/channel.cc + src/core/lib/surface/channel_create.cc src/core/lib/surface/channel_init.cc - src/core/lib/surface/channel_ping.cc src/core/lib/surface/channel_stack_type.cc src/core/lib/surface/completion_queue.cc src/core/lib/surface/completion_queue_factory.cc @@ -2469,6 +2467,7 @@ add_library(grpc src/core/lib/surface/init.cc src/core/lib/surface/init_internally.cc src/core/lib/surface/lame_client.cc + src/core/lib/surface/legacy_channel.cc src/core/lib/surface/metadata_array.cc src/core/lib/surface/server.cc src/core/lib/surface/validate_metadata.cc @@ -2879,7 +2878,6 @@ endif() add_library(grpc_unsecure src/core/client_channel/backup_poller.cc - src/core/client_channel/channel_connectivity.cc src/core/client_channel/client_channel_channelz.cc src/core/client_channel/client_channel_factory.cc src/core/client_channel/client_channel_filter.cc @@ -3184,15 +3182,14 @@ add_library(grpc_unsecure src/core/lib/slice/slice_refcount.cc src/core/lib/slice/slice_string_helpers.cc src/core/lib/surface/api_trace.cc - src/core/lib/surface/builtins.cc src/core/lib/surface/byte_buffer.cc src/core/lib/surface/byte_buffer_reader.cc src/core/lib/surface/call.cc src/core/lib/surface/call_details.cc src/core/lib/surface/call_log_batch.cc src/core/lib/surface/channel.cc + src/core/lib/surface/channel_create.cc src/core/lib/surface/channel_init.cc - src/core/lib/surface/channel_ping.cc src/core/lib/surface/channel_stack_type.cc src/core/lib/surface/completion_queue.cc src/core/lib/surface/completion_queue_factory.cc @@ -3200,6 +3197,7 @@ add_library(grpc_unsecure src/core/lib/surface/init.cc src/core/lib/surface/init_internally.cc src/core/lib/surface/lame_client.cc + src/core/lib/surface/legacy_channel.cc src/core/lib/surface/metadata_array.cc src/core/lib/surface/server.cc src/core/lib/surface/validate_metadata.cc @@ -5111,7 +5109,6 @@ add_library(grpc_authorization_provider src/core/lib/channel/connected_channel.cc src/core/lib/channel/metrics.cc src/core/lib/channel/promise_based_filter.cc - src/core/lib/channel/server_call_tracer_filter.cc src/core/lib/channel/status_util.cc src/core/lib/compression/compression.cc src/core/lib/compression/compression_internal.cc @@ -5312,7 +5309,6 @@ add_library(grpc_authorization_provider src/core/lib/slice/slice_refcount.cc src/core/lib/slice/slice_string_helpers.cc src/core/lib/surface/api_trace.cc - src/core/lib/surface/builtins.cc src/core/lib/surface/byte_buffer.cc src/core/lib/surface/byte_buffer_reader.cc src/core/lib/surface/call.cc @@ -5320,7 +5316,6 @@ add_library(grpc_authorization_provider src/core/lib/surface/call_log_batch.cc src/core/lib/surface/channel.cc src/core/lib/surface/channel_init.cc - src/core/lib/surface/channel_ping.cc src/core/lib/surface/channel_stack_type.cc src/core/lib/surface/completion_queue.cc src/core/lib/surface/completion_queue_factory.cc @@ -5328,7 +5323,6 @@ add_library(grpc_authorization_provider src/core/lib/surface/init_internally.cc src/core/lib/surface/lame_client.cc src/core/lib/surface/metadata_array.cc - src/core/lib/surface/server.cc src/core/lib/surface/validate_metadata.cc src/core/lib/surface/version.cc src/core/lib/surface/wait_for_cq_end_op.cc @@ -8438,6 +8432,7 @@ add_executable(call_filters_test src/core/ext/upb-gen/google/protobuf/descriptor.upb_minitable.c src/core/ext/upb-gen/google/rpc/status.upb_minitable.c src/core/lib/channel/channel_args.cc + src/core/lib/compression/compression.cc src/core/lib/compression/compression_internal.cc src/core/lib/debug/trace.cc src/core/lib/experiments/config.cc @@ -8462,6 +8457,7 @@ add_executable(call_filters_test src/core/lib/slice/slice_buffer.cc src/core/lib/slice/slice_refcount.cc src/core/lib/slice/slice_string_helpers.cc + src/core/lib/surface/api_trace.cc src/core/lib/surface/channel_stack_type.cc src/core/lib/transport/call_filters.cc src/core/lib/transport/call_final_info.cc diff --git a/Makefile b/Makefile index 80515ec9a8d..77c838096dd 100644 --- a/Makefile +++ b/Makefile @@ -959,7 +959,6 @@ endif # transitive_deps: ['address_sorting', 'gpr', 'grpc_abseil', 'cares', 'z', 're2', 'upb_textformat_lib', 'upb_json_lib', 'utf8_range_lib', 'upb_message_lib', 'upb_mem_lib', 'upb_base_lib', 'libssl'] LIBGRPC_SRC = \ src/core/client_channel/backup_poller.cc \ - src/core/client_channel/channel_connectivity.cc \ src/core/client_channel/client_channel_channelz.cc \ src/core/client_channel/client_channel_factory.cc \ src/core/client_channel/client_channel_filter.cc \ @@ -1634,15 +1633,14 @@ LIBGRPC_SRC = \ src/core/lib/slice/slice_refcount.cc \ src/core/lib/slice/slice_string_helpers.cc \ src/core/lib/surface/api_trace.cc \ - src/core/lib/surface/builtins.cc \ src/core/lib/surface/byte_buffer.cc \ src/core/lib/surface/byte_buffer_reader.cc \ src/core/lib/surface/call.cc \ src/core/lib/surface/call_details.cc \ src/core/lib/surface/call_log_batch.cc \ src/core/lib/surface/channel.cc \ + src/core/lib/surface/channel_create.cc \ src/core/lib/surface/channel_init.cc \ - src/core/lib/surface/channel_ping.cc \ src/core/lib/surface/channel_stack_type.cc \ src/core/lib/surface/completion_queue.cc \ src/core/lib/surface/completion_queue_factory.cc \ @@ -1650,6 +1648,7 @@ LIBGRPC_SRC = \ src/core/lib/surface/init.cc \ src/core/lib/surface/init_internally.cc \ src/core/lib/surface/lame_client.cc \ + src/core/lib/surface/legacy_channel.cc \ src/core/lib/surface/metadata_array.cc \ src/core/lib/surface/server.cc \ src/core/lib/surface/validate_metadata.cc \ @@ -1893,7 +1892,6 @@ endif # transitive_deps: ['address_sorting', 'gpr', 'grpc_abseil', 'cares', 'z', 'utf8_range_lib', 'upb_message_lib', 'upb_mem_lib', 'upb_base_lib'] LIBGRPC_UNSECURE_SRC = \ src/core/client_channel/backup_poller.cc \ - src/core/client_channel/channel_connectivity.cc \ src/core/client_channel/client_channel_channelz.cc \ src/core/client_channel/client_channel_factory.cc \ src/core/client_channel/client_channel_filter.cc \ @@ -2198,15 +2196,14 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/slice/slice_refcount.cc \ src/core/lib/slice/slice_string_helpers.cc \ src/core/lib/surface/api_trace.cc \ - src/core/lib/surface/builtins.cc \ src/core/lib/surface/byte_buffer.cc \ src/core/lib/surface/byte_buffer_reader.cc \ src/core/lib/surface/call.cc \ src/core/lib/surface/call_details.cc \ src/core/lib/surface/call_log_batch.cc \ src/core/lib/surface/channel.cc \ + src/core/lib/surface/channel_create.cc \ src/core/lib/surface/channel_init.cc \ - src/core/lib/surface/channel_ping.cc \ src/core/lib/surface/channel_stack_type.cc \ src/core/lib/surface/completion_queue.cc \ src/core/lib/surface/completion_queue_factory.cc \ @@ -2214,6 +2211,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/surface/init.cc \ src/core/lib/surface/init_internally.cc \ src/core/lib/surface/lame_client.cc \ + src/core/lib/surface/legacy_channel.cc \ src/core/lib/surface/metadata_array.cc \ src/core/lib/surface/server.cc \ src/core/lib/surface/validate_metadata.cc \ diff --git a/Package.swift b/Package.swift index 8a78602f6be..6f480c481de 100644 --- a/Package.swift +++ b/Package.swift @@ -117,7 +117,6 @@ let package = Package( "include/grpc/support/workaround_list.h", "src/core/client_channel/backup_poller.cc", "src/core/client_channel/backup_poller.h", - "src/core/client_channel/channel_connectivity.cc", "src/core/client_channel/client_channel_channelz.cc", "src/core/client_channel/client_channel_channelz.h", "src/core/client_channel/client_channel_factory.cc", @@ -1760,8 +1759,6 @@ let package = Package( "src/core/lib/slice/slice_string_helpers.h", "src/core/lib/surface/api_trace.cc", "src/core/lib/surface/api_trace.h", - "src/core/lib/surface/builtins.cc", - "src/core/lib/surface/builtins.h", "src/core/lib/surface/byte_buffer.cc", "src/core/lib/surface/byte_buffer_reader.cc", "src/core/lib/surface/call.cc", @@ -1772,9 +1769,10 @@ let package = Package( "src/core/lib/surface/call_trace.h", "src/core/lib/surface/channel.cc", "src/core/lib/surface/channel.h", + "src/core/lib/surface/channel_create.cc", + "src/core/lib/surface/channel_create.h", "src/core/lib/surface/channel_init.cc", "src/core/lib/surface/channel_init.h", - "src/core/lib/surface/channel_ping.cc", "src/core/lib/surface/channel_stack_type.cc", "src/core/lib/surface/channel_stack_type.h", "src/core/lib/surface/completion_queue.cc", @@ -1789,9 +1787,12 @@ let package = Package( "src/core/lib/surface/init_internally.h", "src/core/lib/surface/lame_client.cc", "src/core/lib/surface/lame_client.h", + "src/core/lib/surface/legacy_channel.cc", + "src/core/lib/surface/legacy_channel.h", "src/core/lib/surface/metadata_array.cc", "src/core/lib/surface/server.cc", "src/core/lib/surface/server.h", + "src/core/lib/surface/server_interface.h", "src/core/lib/surface/validate_metadata.cc", "src/core/lib/surface/validate_metadata.h", "src/core/lib/surface/version.cc", diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index 2c44a4f10a5..073ea701ec2 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -1111,11 +1111,11 @@ libs: - src/core/lib/slice/slice_refcount.h - src/core/lib/slice/slice_string_helpers.h - src/core/lib/surface/api_trace.h - - src/core/lib/surface/builtins.h - src/core/lib/surface/call.h - src/core/lib/surface/call_test_only.h - src/core/lib/surface/call_trace.h - src/core/lib/surface/channel.h + - src/core/lib/surface/channel_create.h - src/core/lib/surface/channel_init.h - src/core/lib/surface/channel_stack_type.h - src/core/lib/surface/completion_queue.h @@ -1124,7 +1124,9 @@ libs: - src/core/lib/surface/init.h - src/core/lib/surface/init_internally.h - src/core/lib/surface/lame_client.h + - src/core/lib/surface/legacy_channel.h - src/core/lib/surface/server.h + - src/core/lib/surface/server_interface.h - src/core/lib/surface/validate_metadata.h - src/core/lib/surface/wait_for_cq_end_op.h - src/core/lib/transport/batch_builder.h @@ -1236,7 +1238,6 @@ libs: - third_party/xxhash/xxhash.h src: - src/core/client_channel/backup_poller.cc - - src/core/client_channel/channel_connectivity.cc - src/core/client_channel/client_channel_channelz.cc - src/core/client_channel/client_channel_factory.cc - src/core/client_channel/client_channel_filter.cc @@ -1911,15 +1912,14 @@ libs: - src/core/lib/slice/slice_refcount.cc - src/core/lib/slice/slice_string_helpers.cc - src/core/lib/surface/api_trace.cc - - src/core/lib/surface/builtins.cc - src/core/lib/surface/byte_buffer.cc - src/core/lib/surface/byte_buffer_reader.cc - src/core/lib/surface/call.cc - src/core/lib/surface/call_details.cc - src/core/lib/surface/call_log_batch.cc - src/core/lib/surface/channel.cc + - src/core/lib/surface/channel_create.cc - src/core/lib/surface/channel_init.cc - - src/core/lib/surface/channel_ping.cc - src/core/lib/surface/channel_stack_type.cc - src/core/lib/surface/completion_queue.cc - src/core/lib/surface/completion_queue_factory.cc @@ -1927,6 +1927,7 @@ libs: - src/core/lib/surface/init.cc - src/core/lib/surface/init_internally.cc - src/core/lib/surface/lame_client.cc + - src/core/lib/surface/legacy_channel.cc - src/core/lib/surface/metadata_array.cc - src/core/lib/surface/server.cc - src/core/lib/surface/validate_metadata.cc @@ -2577,11 +2578,11 @@ libs: - src/core/lib/slice/slice_refcount.h - src/core/lib/slice/slice_string_helpers.h - src/core/lib/surface/api_trace.h - - src/core/lib/surface/builtins.h - src/core/lib/surface/call.h - src/core/lib/surface/call_test_only.h - src/core/lib/surface/call_trace.h - src/core/lib/surface/channel.h + - src/core/lib/surface/channel_create.h - src/core/lib/surface/channel_init.h - src/core/lib/surface/channel_stack_type.h - src/core/lib/surface/completion_queue.h @@ -2590,7 +2591,9 @@ libs: - src/core/lib/surface/init.h - src/core/lib/surface/init_internally.h - src/core/lib/surface/lame_client.h + - src/core/lib/surface/legacy_channel.h - src/core/lib/surface/server.h + - src/core/lib/surface/server_interface.h - src/core/lib/surface/validate_metadata.h - src/core/lib/surface/wait_for_cq_end_op.h - src/core/lib/transport/batch_builder.h @@ -2692,7 +2695,6 @@ libs: - third_party/upb/upb/wire/types.h src: - src/core/client_channel/backup_poller.cc - - src/core/client_channel/channel_connectivity.cc - src/core/client_channel/client_channel_channelz.cc - src/core/client_channel/client_channel_factory.cc - src/core/client_channel/client_channel_filter.cc @@ -2997,15 +2999,14 @@ libs: - src/core/lib/slice/slice_refcount.cc - src/core/lib/slice/slice_string_helpers.cc - src/core/lib/surface/api_trace.cc - - src/core/lib/surface/builtins.cc - src/core/lib/surface/byte_buffer.cc - src/core/lib/surface/byte_buffer_reader.cc - src/core/lib/surface/call.cc - src/core/lib/surface/call_details.cc - src/core/lib/surface/call_log_batch.cc - src/core/lib/surface/channel.cc + - src/core/lib/surface/channel_create.cc - src/core/lib/surface/channel_init.cc - - src/core/lib/surface/channel_ping.cc - src/core/lib/surface/channel_stack_type.cc - src/core/lib/surface/completion_queue.cc - src/core/lib/surface/completion_queue_factory.cc @@ -3013,6 +3014,7 @@ libs: - src/core/lib/surface/init.cc - src/core/lib/surface/init_internally.cc - src/core/lib/surface/lame_client.cc + - src/core/lib/surface/legacy_channel.cc - src/core/lib/surface/metadata_array.cc - src/core/lib/surface/server.cc - src/core/lib/surface/validate_metadata.cc @@ -4654,7 +4656,6 @@ libs: - src/core/lib/slice/slice_refcount.h - src/core/lib/slice/slice_string_helpers.h - src/core/lib/surface/api_trace.h - - src/core/lib/surface/builtins.h - src/core/lib/surface/call.h - src/core/lib/surface/call_test_only.h - src/core/lib/surface/call_trace.h @@ -4667,7 +4668,7 @@ libs: - src/core/lib/surface/init.h - src/core/lib/surface/init_internally.h - src/core/lib/surface/lame_client.h - - src/core/lib/surface/server.h + - src/core/lib/surface/server_interface.h - src/core/lib/surface/validate_metadata.h - src/core/lib/surface/wait_for_cq_end_op.h - src/core/lib/transport/batch_builder.h @@ -4756,7 +4757,6 @@ libs: - src/core/lib/channel/connected_channel.cc - src/core/lib/channel/metrics.cc - src/core/lib/channel/promise_based_filter.cc - - src/core/lib/channel/server_call_tracer_filter.cc - src/core/lib/channel/status_util.cc - src/core/lib/compression/compression.cc - src/core/lib/compression/compression_internal.cc @@ -4957,7 +4957,6 @@ libs: - src/core/lib/slice/slice_refcount.cc - src/core/lib/slice/slice_string_helpers.cc - src/core/lib/surface/api_trace.cc - - src/core/lib/surface/builtins.cc - src/core/lib/surface/byte_buffer.cc - src/core/lib/surface/byte_buffer_reader.cc - src/core/lib/surface/call.cc @@ -4965,7 +4964,6 @@ libs: - src/core/lib/surface/call_log_batch.cc - src/core/lib/surface/channel.cc - src/core/lib/surface/channel_init.cc - - src/core/lib/surface/channel_ping.cc - src/core/lib/surface/channel_stack_type.cc - src/core/lib/surface/completion_queue.cc - src/core/lib/surface/completion_queue_factory.cc @@ -4973,7 +4971,6 @@ libs: - src/core/lib/surface/init_internally.cc - src/core/lib/surface/lame_client.cc - src/core/lib/surface/metadata_array.cc - - src/core/lib/surface/server.cc - src/core/lib/surface/validate_metadata.cc - src/core/lib/surface/version.cc - src/core/lib/surface/wait_for_cq_end_op.cc @@ -6407,6 +6404,7 @@ targets: - src/core/lib/slice/slice_internal.h - src/core/lib/slice/slice_refcount.h - src/core/lib/slice/slice_string_helpers.h + - src/core/lib/surface/api_trace.h - src/core/lib/surface/channel_stack_type.h - src/core/lib/transport/call_filters.h - src/core/lib/transport/call_final_info.h @@ -6447,6 +6445,7 @@ targets: - src/core/ext/upb-gen/google/protobuf/descriptor.upb_minitable.c - src/core/ext/upb-gen/google/rpc/status.upb_minitable.c - src/core/lib/channel/channel_args.cc + - src/core/lib/compression/compression.cc - src/core/lib/compression/compression_internal.cc - src/core/lib/debug/trace.cc - src/core/lib/experiments/config.cc @@ -6471,6 +6470,7 @@ targets: - src/core/lib/slice/slice_buffer.cc - src/core/lib/slice/slice_refcount.cc - src/core/lib/slice/slice_string_helpers.cc + - src/core/lib/surface/api_trace.cc - src/core/lib/surface/channel_stack_type.cc - src/core/lib/transport/call_filters.cc - src/core/lib/transport/call_final_info.cc diff --git a/config.m4 b/config.m4 index a40bebe4b62..950fdbb4632 100644 --- a/config.m4 +++ b/config.m4 @@ -42,7 +42,6 @@ if test "$PHP_GRPC" != "no"; then PHP_NEW_EXTENSION(grpc, src/core/client_channel/backup_poller.cc \ - src/core/client_channel/channel_connectivity.cc \ src/core/client_channel/client_channel_channelz.cc \ src/core/client_channel/client_channel_factory.cc \ src/core/client_channel/client_channel_filter.cc \ @@ -762,15 +761,14 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/slice/slice_refcount.cc \ src/core/lib/slice/slice_string_helpers.cc \ src/core/lib/surface/api_trace.cc \ - src/core/lib/surface/builtins.cc \ src/core/lib/surface/byte_buffer.cc \ src/core/lib/surface/byte_buffer_reader.cc \ src/core/lib/surface/call.cc \ src/core/lib/surface/call_details.cc \ src/core/lib/surface/call_log_batch.cc \ src/core/lib/surface/channel.cc \ + src/core/lib/surface/channel_create.cc \ src/core/lib/surface/channel_init.cc \ - src/core/lib/surface/channel_ping.cc \ src/core/lib/surface/channel_stack_type.cc \ src/core/lib/surface/completion_queue.cc \ src/core/lib/surface/completion_queue_factory.cc \ @@ -778,6 +776,7 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/surface/init.cc \ src/core/lib/surface/init_internally.cc \ src/core/lib/surface/lame_client.cc \ + src/core/lib/surface/legacy_channel.cc \ src/core/lib/surface/metadata_array.cc \ src/core/lib/surface/server.cc \ src/core/lib/surface/validate_metadata.cc \ diff --git a/config.w32 b/config.w32 index bebb0341fa7..59efe403184 100644 --- a/config.w32 +++ b/config.w32 @@ -7,7 +7,6 @@ if (PHP_GRPC != "no") { EXTENSION("grpc", "src\\core\\client_channel\\backup_poller.cc " + - "src\\core\\client_channel\\channel_connectivity.cc " + "src\\core\\client_channel\\client_channel_channelz.cc " + "src\\core\\client_channel\\client_channel_factory.cc " + "src\\core\\client_channel\\client_channel_filter.cc " + @@ -727,15 +726,14 @@ if (PHP_GRPC != "no") { "src\\core\\lib\\slice\\slice_refcount.cc " + "src\\core\\lib\\slice\\slice_string_helpers.cc " + "src\\core\\lib\\surface\\api_trace.cc " + - "src\\core\\lib\\surface\\builtins.cc " + "src\\core\\lib\\surface\\byte_buffer.cc " + "src\\core\\lib\\surface\\byte_buffer_reader.cc " + "src\\core\\lib\\surface\\call.cc " + "src\\core\\lib\\surface\\call_details.cc " + "src\\core\\lib\\surface\\call_log_batch.cc " + "src\\core\\lib\\surface\\channel.cc " + + "src\\core\\lib\\surface\\channel_create.cc " + "src\\core\\lib\\surface\\channel_init.cc " + - "src\\core\\lib\\surface\\channel_ping.cc " + "src\\core\\lib\\surface\\channel_stack_type.cc " + "src\\core\\lib\\surface\\completion_queue.cc " + "src\\core\\lib\\surface\\completion_queue_factory.cc " + @@ -743,6 +741,7 @@ if (PHP_GRPC != "no") { "src\\core\\lib\\surface\\init.cc " + "src\\core\\lib\\surface\\init_internally.cc " + "src\\core\\lib\\surface\\lame_client.cc " + + "src\\core\\lib\\surface\\legacy_channel.cc " + "src\\core\\lib\\surface\\metadata_array.cc " + "src\\core\\lib\\surface\\server.cc " + "src\\core\\lib\\surface\\validate_metadata.cc " + diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index 7fd6798f3cf..a03916faa25 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -1217,11 +1217,11 @@ Pod::Spec.new do |s| 'src/core/lib/slice/slice_refcount.h', 'src/core/lib/slice/slice_string_helpers.h', 'src/core/lib/surface/api_trace.h', - 'src/core/lib/surface/builtins.h', 'src/core/lib/surface/call.h', 'src/core/lib/surface/call_test_only.h', 'src/core/lib/surface/call_trace.h', 'src/core/lib/surface/channel.h', + 'src/core/lib/surface/channel_create.h', 'src/core/lib/surface/channel_init.h', 'src/core/lib/surface/channel_stack_type.h', 'src/core/lib/surface/completion_queue.h', @@ -1230,7 +1230,9 @@ Pod::Spec.new do |s| 'src/core/lib/surface/init.h', 'src/core/lib/surface/init_internally.h', 'src/core/lib/surface/lame_client.h', + 'src/core/lib/surface/legacy_channel.h', 'src/core/lib/surface/server.h', + 'src/core/lib/surface/server_interface.h', 'src/core/lib/surface/validate_metadata.h', 'src/core/lib/surface/wait_for_cq_end_op.h', 'src/core/lib/transport/batch_builder.h', @@ -2476,11 +2478,11 @@ Pod::Spec.new do |s| 'src/core/lib/slice/slice_refcount.h', 'src/core/lib/slice/slice_string_helpers.h', 'src/core/lib/surface/api_trace.h', - 'src/core/lib/surface/builtins.h', 'src/core/lib/surface/call.h', 'src/core/lib/surface/call_test_only.h', 'src/core/lib/surface/call_trace.h', 'src/core/lib/surface/channel.h', + 'src/core/lib/surface/channel_create.h', 'src/core/lib/surface/channel_init.h', 'src/core/lib/surface/channel_stack_type.h', 'src/core/lib/surface/completion_queue.h', @@ -2489,7 +2491,9 @@ Pod::Spec.new do |s| 'src/core/lib/surface/init.h', 'src/core/lib/surface/init_internally.h', 'src/core/lib/surface/lame_client.h', + 'src/core/lib/surface/legacy_channel.h', 'src/core/lib/surface/server.h', + 'src/core/lib/surface/server_interface.h', 'src/core/lib/surface/validate_metadata.h', 'src/core/lib/surface/wait_for_cq_end_op.h', 'src/core/lib/transport/batch_builder.h', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 0d0d8298302..d301e36c83b 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -231,7 +231,6 @@ Pod::Spec.new do |s| ss.source_files = 'src/core/client_channel/backup_poller.cc', 'src/core/client_channel/backup_poller.h', - 'src/core/client_channel/channel_connectivity.cc', 'src/core/client_channel/client_channel_channelz.cc', 'src/core/client_channel/client_channel_channelz.h', 'src/core/client_channel/client_channel_factory.cc', @@ -1870,8 +1869,6 @@ Pod::Spec.new do |s| 'src/core/lib/slice/slice_string_helpers.h', 'src/core/lib/surface/api_trace.cc', 'src/core/lib/surface/api_trace.h', - 'src/core/lib/surface/builtins.cc', - 'src/core/lib/surface/builtins.h', 'src/core/lib/surface/byte_buffer.cc', 'src/core/lib/surface/byte_buffer_reader.cc', 'src/core/lib/surface/call.cc', @@ -1882,9 +1879,10 @@ Pod::Spec.new do |s| 'src/core/lib/surface/call_trace.h', 'src/core/lib/surface/channel.cc', 'src/core/lib/surface/channel.h', + 'src/core/lib/surface/channel_create.cc', + 'src/core/lib/surface/channel_create.h', 'src/core/lib/surface/channel_init.cc', 'src/core/lib/surface/channel_init.h', - 'src/core/lib/surface/channel_ping.cc', 'src/core/lib/surface/channel_stack_type.cc', 'src/core/lib/surface/channel_stack_type.h', 'src/core/lib/surface/completion_queue.cc', @@ -1899,9 +1897,12 @@ Pod::Spec.new do |s| 'src/core/lib/surface/init_internally.h', 'src/core/lib/surface/lame_client.cc', 'src/core/lib/surface/lame_client.h', + 'src/core/lib/surface/legacy_channel.cc', + 'src/core/lib/surface/legacy_channel.h', 'src/core/lib/surface/metadata_array.cc', 'src/core/lib/surface/server.cc', 'src/core/lib/surface/server.h', + 'src/core/lib/surface/server_interface.h', 'src/core/lib/surface/validate_metadata.cc', 'src/core/lib/surface/validate_metadata.h', 'src/core/lib/surface/version.cc', @@ -3257,11 +3258,11 @@ Pod::Spec.new do |s| 'src/core/lib/slice/slice_refcount.h', 'src/core/lib/slice/slice_string_helpers.h', 'src/core/lib/surface/api_trace.h', - 'src/core/lib/surface/builtins.h', 'src/core/lib/surface/call.h', 'src/core/lib/surface/call_test_only.h', 'src/core/lib/surface/call_trace.h', 'src/core/lib/surface/channel.h', + 'src/core/lib/surface/channel_create.h', 'src/core/lib/surface/channel_init.h', 'src/core/lib/surface/channel_stack_type.h', 'src/core/lib/surface/completion_queue.h', @@ -3270,7 +3271,9 @@ Pod::Spec.new do |s| 'src/core/lib/surface/init.h', 'src/core/lib/surface/init_internally.h', 'src/core/lib/surface/lame_client.h', + 'src/core/lib/surface/legacy_channel.h', 'src/core/lib/surface/server.h', + 'src/core/lib/surface/server_interface.h', 'src/core/lib/surface/validate_metadata.h', 'src/core/lib/surface/wait_for_cq_end_op.h', 'src/core/lib/transport/batch_builder.h', diff --git a/grpc.gemspec b/grpc.gemspec index a0ea7c116de..f31e053b11d 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -123,7 +123,6 @@ Gem::Specification.new do |s| s.files += %w( include/grpc/support/workaround_list.h ) s.files += %w( src/core/client_channel/backup_poller.cc ) s.files += %w( src/core/client_channel/backup_poller.h ) - s.files += %w( src/core/client_channel/channel_connectivity.cc ) s.files += %w( src/core/client_channel/client_channel_channelz.cc ) s.files += %w( src/core/client_channel/client_channel_channelz.h ) s.files += %w( src/core/client_channel/client_channel_factory.cc ) @@ -1762,8 +1761,6 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/slice/slice_string_helpers.h ) s.files += %w( src/core/lib/surface/api_trace.cc ) s.files += %w( src/core/lib/surface/api_trace.h ) - s.files += %w( src/core/lib/surface/builtins.cc ) - s.files += %w( src/core/lib/surface/builtins.h ) s.files += %w( src/core/lib/surface/byte_buffer.cc ) s.files += %w( src/core/lib/surface/byte_buffer_reader.cc ) s.files += %w( src/core/lib/surface/call.cc ) @@ -1774,9 +1771,10 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/surface/call_trace.h ) s.files += %w( src/core/lib/surface/channel.cc ) s.files += %w( src/core/lib/surface/channel.h ) + s.files += %w( src/core/lib/surface/channel_create.cc ) + s.files += %w( src/core/lib/surface/channel_create.h ) s.files += %w( src/core/lib/surface/channel_init.cc ) s.files += %w( src/core/lib/surface/channel_init.h ) - s.files += %w( src/core/lib/surface/channel_ping.cc ) s.files += %w( src/core/lib/surface/channel_stack_type.cc ) s.files += %w( src/core/lib/surface/channel_stack_type.h ) s.files += %w( src/core/lib/surface/completion_queue.cc ) @@ -1791,9 +1789,12 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/surface/init_internally.h ) s.files += %w( src/core/lib/surface/lame_client.cc ) s.files += %w( src/core/lib/surface/lame_client.h ) + s.files += %w( src/core/lib/surface/legacy_channel.cc ) + s.files += %w( src/core/lib/surface/legacy_channel.h ) s.files += %w( src/core/lib/surface/metadata_array.cc ) s.files += %w( src/core/lib/surface/server.cc ) s.files += %w( src/core/lib/surface/server.h ) + s.files += %w( src/core/lib/surface/server_interface.h ) s.files += %w( src/core/lib/surface/validate_metadata.cc ) s.files += %w( src/core/lib/surface/validate_metadata.h ) s.files += %w( src/core/lib/surface/version.cc ) diff --git a/grpc.gyp b/grpc.gyp index ebd5273440f..fcffddc3163 100644 --- a/grpc.gyp +++ b/grpc.gyp @@ -274,7 +274,6 @@ ], 'sources': [ 'src/core/client_channel/backup_poller.cc', - 'src/core/client_channel/channel_connectivity.cc', 'src/core/client_channel/client_channel_channelz.cc', 'src/core/client_channel/client_channel_factory.cc', 'src/core/client_channel/client_channel_filter.cc', @@ -949,15 +948,14 @@ 'src/core/lib/slice/slice_refcount.cc', 'src/core/lib/slice/slice_string_helpers.cc', 'src/core/lib/surface/api_trace.cc', - 'src/core/lib/surface/builtins.cc', 'src/core/lib/surface/byte_buffer.cc', 'src/core/lib/surface/byte_buffer_reader.cc', 'src/core/lib/surface/call.cc', 'src/core/lib/surface/call_details.cc', 'src/core/lib/surface/call_log_batch.cc', 'src/core/lib/surface/channel.cc', + 'src/core/lib/surface/channel_create.cc', 'src/core/lib/surface/channel_init.cc', - 'src/core/lib/surface/channel_ping.cc', 'src/core/lib/surface/channel_stack_type.cc', 'src/core/lib/surface/completion_queue.cc', 'src/core/lib/surface/completion_queue_factory.cc', @@ -965,6 +963,7 @@ 'src/core/lib/surface/init.cc', 'src/core/lib/surface/init_internally.cc', 'src/core/lib/surface/lame_client.cc', + 'src/core/lib/surface/legacy_channel.cc', 'src/core/lib/surface/metadata_array.cc', 'src/core/lib/surface/server.cc', 'src/core/lib/surface/validate_metadata.cc', @@ -1149,7 +1148,6 @@ ], 'sources': [ 'src/core/client_channel/backup_poller.cc', - 'src/core/client_channel/channel_connectivity.cc', 'src/core/client_channel/client_channel_channelz.cc', 'src/core/client_channel/client_channel_factory.cc', 'src/core/client_channel/client_channel_filter.cc', @@ -1454,15 +1452,14 @@ 'src/core/lib/slice/slice_refcount.cc', 'src/core/lib/slice/slice_string_helpers.cc', 'src/core/lib/surface/api_trace.cc', - 'src/core/lib/surface/builtins.cc', 'src/core/lib/surface/byte_buffer.cc', 'src/core/lib/surface/byte_buffer_reader.cc', 'src/core/lib/surface/call.cc', 'src/core/lib/surface/call_details.cc', 'src/core/lib/surface/call_log_batch.cc', 'src/core/lib/surface/channel.cc', + 'src/core/lib/surface/channel_create.cc', 'src/core/lib/surface/channel_init.cc', - 'src/core/lib/surface/channel_ping.cc', 'src/core/lib/surface/channel_stack_type.cc', 'src/core/lib/surface/completion_queue.cc', 'src/core/lib/surface/completion_queue_factory.cc', @@ -1470,6 +1467,7 @@ 'src/core/lib/surface/init.cc', 'src/core/lib/surface/init_internally.cc', 'src/core/lib/surface/lame_client.cc', + 'src/core/lib/surface/legacy_channel.cc', 'src/core/lib/surface/metadata_array.cc', 'src/core/lib/surface/server.cc', 'src/core/lib/surface/validate_metadata.cc', @@ -2063,7 +2061,6 @@ 'src/core/lib/channel/connected_channel.cc', 'src/core/lib/channel/metrics.cc', 'src/core/lib/channel/promise_based_filter.cc', - 'src/core/lib/channel/server_call_tracer_filter.cc', 'src/core/lib/channel/status_util.cc', 'src/core/lib/compression/compression.cc', 'src/core/lib/compression/compression_internal.cc', @@ -2264,7 +2261,6 @@ 'src/core/lib/slice/slice_refcount.cc', 'src/core/lib/slice/slice_string_helpers.cc', 'src/core/lib/surface/api_trace.cc', - 'src/core/lib/surface/builtins.cc', 'src/core/lib/surface/byte_buffer.cc', 'src/core/lib/surface/byte_buffer_reader.cc', 'src/core/lib/surface/call.cc', @@ -2272,7 +2268,6 @@ 'src/core/lib/surface/call_log_batch.cc', 'src/core/lib/surface/channel.cc', 'src/core/lib/surface/channel_init.cc', - 'src/core/lib/surface/channel_ping.cc', 'src/core/lib/surface/channel_stack_type.cc', 'src/core/lib/surface/completion_queue.cc', 'src/core/lib/surface/completion_queue_factory.cc', @@ -2280,7 +2275,6 @@ 'src/core/lib/surface/init_internally.cc', 'src/core/lib/surface/lame_client.cc', 'src/core/lib/surface/metadata_array.cc', - 'src/core/lib/surface/server.cc', 'src/core/lib/surface/validate_metadata.cc', 'src/core/lib/surface/version.cc', 'src/core/lib/surface/wait_for_cq_end_op.cc', diff --git a/package.xml b/package.xml index caa7395a997..42ca1197a08 100644 --- a/package.xml +++ b/package.xml @@ -105,7 +105,6 @@ - @@ -1744,8 +1743,6 @@ - - @@ -1756,9 +1753,10 @@ + + - @@ -1773,9 +1771,12 @@ + + + diff --git a/src/core/BUILD b/src/core/BUILD index d329d8c287d..185aa98a554 100644 --- a/src/core/BUILD +++ b/src/core/BUILD @@ -113,6 +113,35 @@ grpc_cc_library( language = "c++", ) +grpc_cc_library( + name = "server_call_tracer_filter", + srcs = [ + "lib/channel/server_call_tracer_filter.cc", + ], + external_deps = [ + "absl/status", + "absl/status:statusor", + ], + language = "c++", + visibility = ["@grpc:alt_grpc_base_legacy"], + deps = [ + "arena_promise", + "call_finalization", + "cancel_callback", + "channel_args", + "channel_fwd", + "channel_stack_type", + "context", + "map", + "pipe", + "//:call_tracer", + "//:config", + "//:gpr_platform", + "//:grpc_base", + "//:legacy_context", + ], +) + grpc_cc_library( name = "atomic_utils", language = "c++", @@ -137,6 +166,7 @@ grpc_cc_library( "hpack_constants", "metadata_batch", "slice", + "//:call_tracer", "//:gpr_platform", "//:grpc_base", ], @@ -2526,10 +2556,10 @@ grpc_cc_library( grpc_cc_library( name = "channel_args_endpoint_config", srcs = [ - "//src/core:lib/event_engine/channel_args_endpoint_config.cc", + "lib/event_engine/channel_args_endpoint_config.cc", ], hdrs = [ - "//src/core:lib/event_engine/channel_args_endpoint_config.h", + "lib/event_engine/channel_args_endpoint_config.h", ], external_deps = [ "absl/strings", @@ -2889,6 +2919,19 @@ grpc_cc_library( ], ) +grpc_cc_library( + name = "server_interface", + hdrs = [ + "lib/surface/server_interface.h", + ], + language = "c++", + deps = [ + "channel_args", + "//:channelz", + "//:gpr_platform", + ], +) + grpc_cc_library( name = "single_set_ptr", hdrs = [ @@ -3005,6 +3048,7 @@ grpc_cc_library( "grpc_service_config", "lb_policy", "unique_type_name", + "//:call_tracer", "//:gpr_public_hdrs", "//:grpc_base", "//:legacy_context", @@ -3827,6 +3871,7 @@ grpc_cc_library( "iomgr_fwd", "unique_type_name", "useful", + "//:api_trace", "//:channel_arg_names", "//:debug_location", "//:exec_ctx", @@ -3882,6 +3927,7 @@ grpc_cc_library( "unique_type_name", "useful", "//:alts_util", + "//:api_trace", "//:channel_arg_names", "//:exec_ctx", "//:gpr", @@ -3955,6 +4001,7 @@ grpc_cc_library( "status_helper", "unique_type_name", "useful", + "//:api_trace", "//:channel_arg_names", "//:debug_location", "//:exec_ctx", @@ -3995,6 +4042,7 @@ grpc_cc_library( "slice", "unique_type_name", "useful", + "//:api_trace", "//:exec_ctx", "//:gpr", "//:grpc_base", @@ -4042,6 +4090,7 @@ grpc_cc_library( "time", "unique_type_name", "useful", + "//:api_trace", "//:gpr", "//:grpc_base", "//:grpc_credentials_util", @@ -4687,7 +4736,9 @@ grpc_cc_library( "useful", "validation_errors", "//:backoff", + "//:channel", "//:channel_arg_names", + "//:channel_create", "//:channelz", "//:config", "//:debug_location", @@ -4786,7 +4837,9 @@ grpc_cc_library( "time", "validation_errors", "//:backoff", + "//:channel", "//:channel_arg_names", + "//:channel_create", "//:channelz", "//:config", "//:debug_location", @@ -5003,7 +5056,9 @@ grpc_cc_library( "validation_errors", "xds_type_upb", "xds_type_upbdefs", + "//:channel", "//:channel_arg_names", + "//:channel_create", "//:config", "//:debug_location", "//:endpoint_addresses", @@ -5084,6 +5139,7 @@ grpc_cc_library( "resolved_address", "slice_refcount", "unique_type_name", + "//:api_trace", "//:config", "//:debug_location", "//:exec_ctx", @@ -5096,6 +5152,7 @@ grpc_cc_library( "//:iomgr", "//:parse_address", "//:ref_counted_ptr", + "//:server", "//:sockaddr_utils", "//:uri_parser", "//:xds_client", @@ -5969,6 +6026,7 @@ grpc_cc_library( language = "c++", deps = [ "arena_promise", + "call_finalization", "channel_args", "channel_fwd", "channel_stack_type", @@ -6636,7 +6694,10 @@ grpc_cc_library( "tcp_connect_handshaker", "time", "unique_type_name", + "//:api_trace", + "//:channel", "//:channel_arg_names", + "//:channel_create", "//:channelz", "//:config", "//:debug_location", @@ -6690,6 +6751,7 @@ grpc_cc_library( "status_helper", "time", "unique_type_name", + "//:api_trace", "//:channel_arg_names", "//:channelz", "//:chttp2_legacy_frame", @@ -6705,6 +6767,7 @@ grpc_cc_library( "//:iomgr", "//:orphanable", "//:ref_counted_ptr", + "//:server", "//:sockaddr_utils", "//:uri_parser", ], @@ -6744,7 +6807,10 @@ grpc_cc_library( "status_helper", "time", "try_seq", + "//:api_trace", + "//:channel", "//:channel_arg_names", + "//:channel_create", "//:channelz", "//:config", "//:debug_location", @@ -6756,6 +6822,7 @@ grpc_cc_library( "//:iomgr", "//:promise", "//:ref_counted_ptr", + "//:server", ], ) @@ -6916,6 +6983,7 @@ grpc_cc_library( "slice", "slice_buffer", "time", + "//:call_tracer", "//:channel_arg_names", "//:config", "//:gpr", @@ -7112,6 +7180,21 @@ grpc_cc_library( ], ) +grpc_cc_library( + name = "call_finalization", + hdrs = [ + "lib/channel/call_finalization.h", + ], + language = "c++", + visibility = ["@grpc:alt_grpc_base_legacy"], + deps = [ + "arena", + "call_final_info", + "context", + "//:gpr_platform", + ], +) + grpc_cc_library( name = "call_filters", srcs = [ @@ -7266,7 +7349,7 @@ grpc_cc_library( deps = [ "arena", "chunked_vector", - "compression_internal", + "compression", "experiments", "if_list", "metadata_compression_traits", @@ -7313,8 +7396,9 @@ grpc_cc_library( ) grpc_cc_library( - name = "compression_internal", + name = "compression", srcs = [ + "lib/compression/compression.cc", "lib/compression/compression_internal.cc", ], hdrs = [ @@ -7331,6 +7415,8 @@ grpc_cc_library( "channel_args", "ref_counted_string", "slice", + "useful", + "//:api_trace", "//:gpr", "//:grpc_public_hdrs", "//:grpc_trace", @@ -7397,6 +7483,7 @@ grpc_cc_library( "//:iomgr", "//:orphanable", "//:ref_counted_ptr", + "//:server", ], ) @@ -7446,6 +7533,9 @@ grpc_cc_library( "time", "try_seq", "wait_for_callback", + "//:api_trace", + "//:channel", + "//:channel_create", "//:debug_location", "//:exec_ctx", "//:gpr", diff --git a/src/core/client_channel/channel_connectivity.cc b/src/core/client_channel/channel_connectivity.cc deleted file mode 100644 index 5ae94d612a5..00000000000 --- a/src/core/client_channel/channel_connectivity.cc +++ /dev/null @@ -1,250 +0,0 @@ -// -// Copyright 2015 gRPC authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -#include - -#include - -#include "absl/base/thread_annotations.h" -#include "absl/status/status.h" -#include "absl/types/optional.h" - -#include -#include -#include -#include -#include - -#include "src/core/client_channel/client_channel_filter.h" -#include "src/core/lib/channel/channel_fwd.h" -#include "src/core/lib/channel/channel_stack.h" -#include "src/core/lib/debug/trace.h" -#include "src/core/lib/gprpp/crash.h" -#include "src/core/lib/gprpp/dual_ref_counted.h" -#include "src/core/lib/gprpp/ref_counted_ptr.h" -#include "src/core/lib/gprpp/sync.h" -#include "src/core/lib/gprpp/time.h" -#include "src/core/lib/iomgr/closure.h" -#include "src/core/lib/iomgr/error.h" -#include "src/core/lib/iomgr/exec_ctx.h" -#include "src/core/lib/iomgr/polling_entity.h" -#include "src/core/lib/surface/api_trace.h" -#include "src/core/lib/surface/channel.h" -#include "src/core/lib/surface/completion_queue.h" -#include "src/core/lib/surface/lame_client.h" - -namespace grpc_core { -namespace { - -bool IsLameChannel(Channel* channel) { - grpc_channel_element* elem = - grpc_channel_stack_last_element(channel->channel_stack()); - return elem->filter == &LameClientFilter::kFilter; -} - -} // namespace -} // namespace grpc_core - -grpc_connectivity_state grpc_channel_check_connectivity_state( - grpc_channel* c_channel, int try_to_connect) { - grpc_core::ApplicationCallbackExecCtx callback_exec_ctx; - grpc_core::ExecCtx exec_ctx; - GRPC_API_TRACE( - "grpc_channel_check_connectivity_state(channel=%p, try_to_connect=%d)", 2, - (c_channel, try_to_connect)); - grpc_core::Channel* channel = grpc_core::Channel::FromC(c_channel); - // Forward through to the underlying client channel. - grpc_core::ClientChannelFilter* client_channel = - grpc_core::ClientChannelFilter::GetFromChannel(channel); - if (GPR_UNLIKELY(client_channel == nullptr)) { - if (grpc_core::IsLameChannel(channel)) { - return GRPC_CHANNEL_TRANSIENT_FAILURE; - } - gpr_log(GPR_ERROR, - "grpc_channel_check_connectivity_state called on something that is " - "not a client channel"); - return GRPC_CHANNEL_SHUTDOWN; - } - return client_channel->CheckConnectivityState(try_to_connect); -} - -int grpc_channel_support_connectivity_watcher(grpc_channel* channel) { - return grpc_core::ClientChannelFilter::GetFromChannel( - grpc_core::Channel::FromC(channel)) != nullptr; -} - -namespace grpc_core { -namespace { - -class StateWatcher : public DualRefCounted { - public: - StateWatcher(grpc_channel* c_channel, grpc_completion_queue* cq, void* tag, - grpc_connectivity_state last_observed_state, - gpr_timespec deadline) - : channel_(Channel::FromC(c_channel)->Ref()), - cq_(cq), - tag_(tag), - state_(last_observed_state) { - GPR_ASSERT(grpc_cq_begin_op(cq, tag)); - GRPC_CLOSURE_INIT(&on_complete_, WatchComplete, this, nullptr); - ClientChannelFilter* client_channel = - ClientChannelFilter::GetFromChannel(channel_.get()); - if (client_channel == nullptr) { - // If the target URI used to create the channel was invalid, channel - // stack initialization failed, and that caused us to create a lame - // channel. In that case, connectivity state will never change (it - // will always be TRANSIENT_FAILURE), so we don't actually start a - // watch, but we are hiding that fact from the application. - if (IsLameChannel(channel_.get())) { - // A ref is held by the timer callback. - StartTimer(Timestamp::FromTimespecRoundUp(deadline)); - // Ref from object creation needs to be freed here since lame channel - // does not have a watcher. - Unref(); - return; - } - Crash( - "grpc_channel_watch_connectivity_state called on something that is " - "not a client channel"); - } - // Ref from object creation is held by the watcher callback. - auto* watcher_timer_init_state = new WatcherTimerInitState( - this, Timestamp::FromTimespecRoundUp(deadline)); - client_channel->AddExternalConnectivityWatcher( - grpc_polling_entity_create_from_pollset(grpc_cq_pollset(cq)), &state_, - &on_complete_, watcher_timer_init_state->closure()); - } - - private: - // A fire-and-forget object used to delay starting the timer until the - // ClientChannelFilter actually starts the watch. - class WatcherTimerInitState { - public: - WatcherTimerInitState(StateWatcher* state_watcher, Timestamp deadline) - : state_watcher_(state_watcher), deadline_(deadline) { - GRPC_CLOSURE_INIT(&closure_, WatcherTimerInit, this, nullptr); - } - - grpc_closure* closure() { return &closure_; } - - private: - static void WatcherTimerInit(void* arg, grpc_error_handle /*error*/) { - auto* self = static_cast(arg); - self->state_watcher_->StartTimer(self->deadline_); - delete self; - } - - StateWatcher* state_watcher_; - Timestamp deadline_; - grpc_closure closure_; - }; - - void StartTimer(Timestamp deadline) { - const Duration timeout = deadline - Timestamp::Now(); - MutexLock lock(&mu_); - timer_handle_ = channel_->channel_stack()->EventEngine()->RunAfter( - timeout, [self = Ref()]() mutable { - ApplicationCallbackExecCtx callback_exec_ctx; - ExecCtx exec_ctx; - self->TimeoutComplete(); - // StateWatcher deletion might require an active ExecCtx. - self.reset(); - }); - } - - static void WatchComplete(void* arg, grpc_error_handle error) { - auto* self = static_cast(arg); - if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures)) { - GRPC_LOG_IF_ERROR("watch_completion_error", error); - } - { - MutexLock lock(&self->mu_); - if (self->timer_handle_.has_value()) { - self->channel_->channel_stack()->EventEngine()->Cancel( - *self->timer_handle_); - } - } - // Watcher fired when either notified or cancelled, either way the state of - // this watcher has been cleared from the client channel. Thus there is no - // need to cancel the watch again. - self->Unref(); - } - - void TimeoutComplete() { - timer_fired_ = true; - // If this is a client channel (not a lame channel), cancel the watch. - ClientChannelFilter* client_channel = - ClientChannelFilter::GetFromChannel(channel_.get()); - if (client_channel != nullptr) { - client_channel->CancelExternalConnectivityWatcher(&on_complete_); - } - } - - // Invoked when both strong refs are released. - void Orphan() override { - WeakRef().release(); // Take a weak ref until completion is finished. - grpc_error_handle error = - timer_fired_ - ? GRPC_ERROR_CREATE("Timed out waiting for connection state change") - : absl::OkStatus(); - grpc_cq_end_op(cq_, tag_, error, FinishedCompletion, this, - &completion_storage_); - } - - // Called when the completion is returned to the CQ. - static void FinishedCompletion(void* arg, grpc_cq_completion* /*ignored*/) { - auto* self = static_cast(arg); - self->WeakUnref(); - } - - RefCountedPtr channel_; - grpc_completion_queue* cq_; - void* tag_; - - grpc_connectivity_state state_; - - grpc_cq_completion completion_storage_; - - grpc_closure on_complete_; - - // timer_handle_ might be accessed in parallel from multiple threads, e.g. - // timer callback fired immediately on an EventEngine thread before - // RunAfter() returns. - Mutex mu_; - absl::optional - timer_handle_ ABSL_GUARDED_BY(mu_); - bool timer_fired_ = false; -}; - -} // namespace -} // namespace grpc_core - -void grpc_channel_watch_connectivity_state( - grpc_channel* channel, grpc_connectivity_state last_observed_state, - gpr_timespec deadline, grpc_completion_queue* cq, void* tag) { - grpc_core::ApplicationCallbackExecCtx callback_exec_ctx; - grpc_core::ExecCtx exec_ctx; - GRPC_API_TRACE( - "grpc_channel_watch_connectivity_state(" - "channel=%p, last_observed_state=%d, " - "deadline=gpr_timespec { tv_sec: %" PRId64 - ", tv_nsec: %d, clock_type: %d }, " - "cq=%p, tag=%p)", - 7, - (channel, (int)last_observed_state, deadline.tv_sec, deadline.tv_nsec, - (int)deadline.clock_type, cq, tag)); - new grpc_core::StateWatcher(channel, cq, tag, last_observed_state, deadline); -} diff --git a/src/core/client_channel/client_channel_filter.cc b/src/core/client_channel/client_channel_filter.cc index 5068ed57ba1..c3820ed8846 100644 --- a/src/core/client_channel/client_channel_filter.cc +++ b/src/core/client_channel/client_channel_filter.cc @@ -94,7 +94,6 @@ #include "src/core/lib/slice/slice.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/surface/call.h" -#include "src/core/lib/surface/channel.h" #include "src/core/lib/transport/connectivity_state.h" #include "src/core/lib/transport/error_utils.h" #include "src/core/lib/transport/metadata_batch.h" @@ -1197,16 +1196,6 @@ class ClientChannelFilter::ClientChannelControlHelper // ClientChannelFilter implementation // -ClientChannelFilter* ClientChannelFilter::GetFromChannel(Channel* channel) { - grpc_channel_element* elem = - grpc_channel_stack_last_element(channel->channel_stack()); - if (elem->filter != &kFilterVtableWithPromises && - elem->filter != &kFilterVtableWithoutPromises) { - return nullptr; - } - return static_cast(elem->channel_data); -} - grpc_error_handle ClientChannelFilter::Init(grpc_channel_element* elem, grpc_channel_element_args* args) { GPR_ASSERT(args->is_last); diff --git a/src/core/client_channel/client_channel_filter.h b/src/core/client_channel/client_channel_filter.h index 693366a5b0a..c9d983c2529 100644 --- a/src/core/client_channel/client_channel_filter.h +++ b/src/core/client_channel/client_channel_filter.h @@ -65,7 +65,6 @@ #include "src/core/lib/resource_quota/arena.h" #include "src/core/service_config/service_config.h" #include "src/core/lib/slice/slice.h" -#include "src/core/lib/surface/channel.h" #include "src/core/lib/transport/connectivity_state.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" @@ -115,10 +114,6 @@ class ClientChannelFilter { struct RawPointerChannelArgTag {}; static absl::string_view ChannelArgName() { return GRPC_ARG_CLIENT_CHANNEL; } - // Returns the ClientChannelFilter object from channel, or null if channel - // is not a client channel. - static ClientChannelFilter* GetFromChannel(Channel* channel); - static ArenaPromise MakeCallPromise( grpc_channel_element* elem, CallArgs call_args, NextPromiseFactory next_promise_factory); diff --git a/src/core/ext/transport/binder/client/channel_create_impl.cc b/src/core/ext/transport/binder/client/channel_create_impl.cc index 8a60b56c981..8e86d722f03 100644 --- a/src/core/ext/transport/binder/client/channel_create_impl.cc +++ b/src/core/ext/transport/binder/client/channel_create_impl.cc @@ -28,6 +28,7 @@ #include "src/core/lib/config/core_configuration.h" #include "src/core/lib/surface/api_trace.h" #include "src/core/lib/surface/channel.h" +#include "src/core/lib/surface/channel_create.h" namespace { @@ -56,8 +57,8 @@ grpc_channel* CreateDirectBinderChannelImplForTesting( .PreconditionChannelArgs(args) .Set(GRPC_ARG_DEFAULT_AUTHORITY, "binder.authority"); auto channel = - grpc_core::Channel::Create("binder_target_placeholder", channel_args, - GRPC_CLIENT_DIRECT_CHANNEL, transport); + grpc_core::ChannelCreate("binder_target_placeholder", channel_args, + GRPC_CLIENT_DIRECT_CHANNEL, transport); // TODO(mingcl): Handle error properly GPR_ASSERT(channel.ok()); grpc_channel_args_destroy(args); @@ -75,8 +76,8 @@ grpc_channel* CreateClientBinderChannelImpl(const grpc_channel_args* args) { .SetObject(g_factory); auto channel = - grpc_core::Channel::Create("binder_channel_target_placeholder", - channel_args, GRPC_CLIENT_CHANNEL, nullptr); + grpc_core::ChannelCreate("binder_channel_target_placeholder", + channel_args, GRPC_CLIENT_CHANNEL, nullptr); if (!channel.ok()) { return grpc_lame_client_channel_create( diff --git a/src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc b/src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc index d7c438b1311..adc2aad7dd3 100644 --- a/src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc +++ b/src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc @@ -58,6 +58,7 @@ #include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/surface/api_trace.h" #include "src/core/lib/surface/channel.h" +#include "src/core/lib/surface/channel_create.h" #include "src/core/lib/transport/error_utils.h" #include "src/core/lib/transport/handshaker.h" #include "src/core/lib/transport/promise_endpoint.h" @@ -355,7 +356,7 @@ grpc_channel* grpc_chaotic_good_channel_create(const char* target, std::string canonical_target = grpc_core::CoreConfiguration::Get() .resolver_registry() .AddDefaultPrefixIfNeeded(target); - auto r = grpc_core::Channel::Create( + auto r = grpc_core::ChannelCreate( target, grpc_core::CoreConfiguration::Get() .channel_args_preconditioning() diff --git a/src/core/ext/transport/chttp2/client/chttp2_connector.cc b/src/core/ext/transport/chttp2/client/chttp2_connector.cc index 1d06bfc4cf1..17edffae52a 100644 --- a/src/core/ext/transport/chttp2/client/chttp2_connector.cc +++ b/src/core/ext/transport/chttp2/client/chttp2_connector.cc @@ -64,6 +64,7 @@ #include "src/core/lib/security/security_connector/security_connector.h" #include "src/core/lib/surface/api_trace.h" #include "src/core/lib/surface/channel.h" +#include "src/core/lib/surface/channel_create.h" #include "src/core/lib/surface/channel_stack_type.h" #include "src/core/lib/transport/error_utils.h" #include "src/core/lib/transport/handshaker.h" @@ -307,7 +308,7 @@ class Chttp2SecureClientChannelFactory : public ClientChannelFactory { } }; -absl::StatusOr> CreateChannel(const char* target, +absl::StatusOr> CreateChannel(const char* target, const ChannelArgs& args) { if (target == nullptr) { gpr_log(GPR_ERROR, "cannot create channel with NULL target name"); @@ -317,9 +318,8 @@ absl::StatusOr> CreateChannel(const char* target, std::string canonical_target = CoreConfiguration::Get().resolver_registry().AddDefaultPrefixIfNeeded( target); - return Channel::Create(target, - args.Set(GRPC_ARG_SERVER_URI, canonical_target), - GRPC_CLIENT_CHANNEL, nullptr); + return ChannelCreate(target, args.Set(GRPC_ARG_SERVER_URI, canonical_target), + GRPC_CLIENT_CHANNEL, nullptr); } } // namespace @@ -410,7 +410,7 @@ grpc_channel* grpc_channel_create_from_fd(const char* target, int fd, grpc_core::Transport* transport = grpc_create_chttp2_transport(final_args, client, true); GPR_ASSERT(transport); - auto channel = grpc_core::Channel::Create( + auto channel = grpc_core::ChannelCreate( target, final_args, GRPC_CLIENT_DIRECT_CHANNEL, transport); if (channel.ok()) { grpc_chttp2_transport_start_reading(transport, nullptr, nullptr, nullptr); diff --git a/src/core/ext/transport/cronet/BUILD b/src/core/ext/transport/cronet/BUILD index 7387cda260e..20bbbfc2907 100644 --- a/src/core/ext/transport/cronet/BUILD +++ b/src/core/ext/transport/cronet/BUILD @@ -47,6 +47,8 @@ grpc_cc_library( "client/secure/cronet_channel_create.h", ], deps = [ + "//:channel", + "//:channel_create", "//:grpc_base", "//:grpc_transport_chttp2", "//src/core:channel_args", diff --git a/src/core/ext/transport/cronet/client/secure/cronet_channel_create.cc b/src/core/ext/transport/cronet/client/secure/cronet_channel_create.cc index 996fc0dd42e..5431c576eda 100644 --- a/src/core/ext/transport/cronet/client/secure/cronet_channel_create.cc +++ b/src/core/ext/transport/cronet/client/secure/cronet_channel_create.cc @@ -32,6 +32,7 @@ #include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/surface/channel.h" +#include "src/core/lib/surface/channel_create.h" #include "src/core/lib/surface/channel_stack_type.h" #include "src/core/lib/transport/transport.h" @@ -52,7 +53,7 @@ GRPCAPI grpc_channel* grpc_cronet_secure_channel_create( engine, target, channel_args.ToC().get(), reserved); grpc_core::ExecCtx exec_ctx; - auto channel = grpc_core::Channel::Create(target, channel_args, - GRPC_CLIENT_DIRECT_CHANNEL, ct); + auto channel = grpc_core::ChannelCreate(target, channel_args, + GRPC_CLIENT_DIRECT_CHANNEL, ct); return channel.ok() ? channel->release()->c_ptr() : nullptr; } diff --git a/src/core/ext/transport/inproc/inproc_transport.cc b/src/core/ext/transport/inproc/inproc_transport.cc index 936068b0aa3..6a335cb0386 100644 --- a/src/core/ext/transport/inproc/inproc_transport.cc +++ b/src/core/ext/transport/inproc/inproc_transport.cc @@ -26,6 +26,7 @@ #include "src/core/lib/gprpp/crash.h" #include "src/core/lib/promise/promise.h" #include "src/core/lib/promise/try_seq.h" +#include "src/core/lib/surface/channel_create.h" #include "src/core/lib/surface/server.h" #include "src/core/lib/transport/transport.h" @@ -155,7 +156,7 @@ bool UsePromiseBasedTransport() { return true; } -RefCountedPtr MakeLameChannel(absl::string_view why, +OrphanablePtr MakeLameChannel(absl::string_view why, absl::Status error) { gpr_log(GPR_ERROR, "%s: %s", std::string(why).c_str(), std::string(error.message()).c_str()); @@ -164,11 +165,11 @@ RefCountedPtr MakeLameChannel(absl::string_view why, if (grpc_error_get_int(error, StatusIntProperty::kRpcStatus, &integer)) { status = static_cast(integer); } - return RefCountedPtr(Channel::FromC(grpc_lame_client_channel_create( + return OrphanablePtr(Channel::FromC(grpc_lame_client_channel_create( nullptr, status, std::string(why).c_str()))); } -RefCountedPtr MakeInprocChannel(Server* server, +OrphanablePtr MakeInprocChannel(Server* server, ChannelArgs client_channel_args) { auto transports = MakeInProcessTransportPair(); auto client_transport = std::move(transports.first); @@ -183,7 +184,7 @@ RefCountedPtr MakeInprocChannel(Server* server, return MakeLameChannel("Failed to create server channel", std::move(error)); } std::ignore = server_transport.release(); // consumed by SetupTransport - auto channel = Channel::Create( + auto channel = ChannelCreate( "inproc", client_channel_args.Set(GRPC_ARG_DEFAULT_AUTHORITY, "inproc.authority"), GRPC_CLIENT_DIRECT_CHANNEL, client_transport.release()); diff --git a/src/core/ext/transport/inproc/legacy_inproc_transport.cc b/src/core/ext/transport/inproc/legacy_inproc_transport.cc index 66519608870..b560146daf6 100644 --- a/src/core/ext/transport/inproc/legacy_inproc_transport.cc +++ b/src/core/ext/transport/inproc/legacy_inproc_transport.cc @@ -61,6 +61,7 @@ #include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/surface/api_trace.h" #include "src/core/lib/surface/channel.h" +#include "src/core/lib/surface/channel_create.h" #include "src/core/lib/surface/channel_stack_type.h" #include "src/core/lib/surface/server.h" #include "src/core/lib/transport/connectivity_state.h" @@ -1264,7 +1265,7 @@ grpc_channel* grpc_legacy_inproc_channel_create(grpc_server* server, server_transport, nullptr, server_args, nullptr); grpc_channel* channel = nullptr; if (error.ok()) { - auto new_channel = grpc_core::Channel::Create( + auto new_channel = grpc_core::ChannelCreate( "inproc", client_args, GRPC_CLIENT_DIRECT_CHANNEL, client_transport); if (!new_channel.ok()) { GPR_ASSERT(!channel); diff --git a/src/core/ext/xds/xds_transport_grpc.cc b/src/core/ext/xds/xds_transport_grpc.cc index 136a514e131..c3355c1fe6b 100644 --- a/src/core/ext/xds/xds_transport_grpc.cc +++ b/src/core/ext/xds/xds_transport_grpc.cc @@ -69,15 +69,16 @@ namespace grpc_core { // GrpcXdsTransportFactory::GrpcXdsTransport::GrpcStreamingCall::GrpcStreamingCall( - RefCountedPtr factory, grpc_channel* channel, + RefCountedPtr factory, Channel* channel, const char* method, std::unique_ptr event_handler) : factory_(std::move(factory)), event_handler_(std::move(event_handler)) { // Create call. - call_ = grpc_channel_create_pollset_set_call( - channel, nullptr, GRPC_PROPAGATE_DEFAULTS, factory_->interested_parties(), - StaticSlice::FromStaticString(method).c_slice(), nullptr, - Timestamp::InfFuture(), nullptr); + call_ = channel->CreateCall( + /*parent_call=*/nullptr, GRPC_PROPAGATE_DEFAULTS, /*cq=*/nullptr, + factory_->interested_parties(), Slice::FromStaticString(method), + /*authority=*/absl::nullopt, Timestamp::InfFuture(), + /*registered_method=*/true); GPR_ASSERT(call_ != nullptr); // Init data associated with the call. grpc_metadata_array_init(&initial_metadata_recv_); @@ -252,19 +253,13 @@ class GrpcXdsTransportFactory::GrpcXdsTransport::StateWatcher namespace { -grpc_channel* CreateXdsChannel(const ChannelArgs& args, - const GrpcXdsBootstrap::GrpcXdsServer& server) { +OrphanablePtr CreateXdsChannel( + const ChannelArgs& args, const GrpcXdsBootstrap::GrpcXdsServer& server) { RefCountedPtr channel_creds = CoreConfiguration::Get().channel_creds_registry().CreateChannelCreds( server.channel_creds_config()); - return grpc_channel_create(server.server_uri().c_str(), channel_creds.get(), - args.ToC().get()); -} - -bool IsLameChannel(grpc_channel* channel) { - grpc_channel_element* elem = - grpc_channel_stack_last_element(grpc_channel_get_channel_stack(channel)); - return elem->filter == &LameClientFilter::kFilter; + return OrphanablePtr(Channel::FromC(grpc_channel_create( + server.server_uri().c_str(), channel_creds.get(), args.ToC().get()))); } } // namespace @@ -278,29 +273,19 @@ GrpcXdsTransportFactory::GrpcXdsTransport::GrpcXdsTransport( factory->args_, static_cast(server)); GPR_ASSERT(channel_ != nullptr); - if (IsLameChannel(channel_)) { + if (channel_->IsLame()) { *status = absl::UnavailableError("xds client has a lame channel"); } else { - ClientChannelFilter* client_channel = - ClientChannelFilter::GetFromChannel(Channel::FromC(channel_)); - GPR_ASSERT(client_channel != nullptr); watcher_ = new StateWatcher(std::move(on_connectivity_failure)); - client_channel->AddConnectivityWatcher( + channel_->AddConnectivityWatcher( GRPC_CHANNEL_IDLE, OrphanablePtr(watcher_)); } } -GrpcXdsTransportFactory::GrpcXdsTransport::~GrpcXdsTransport() { - grpc_channel_destroy_internal(channel_); -} - void GrpcXdsTransportFactory::GrpcXdsTransport::Orphan() { - if (!IsLameChannel(channel_)) { - ClientChannelFilter* client_channel = - ClientChannelFilter::GetFromChannel(Channel::FromC(channel_)); - GPR_ASSERT(client_channel != nullptr); - client_channel->RemoveConnectivityWatcher(watcher_); + if (!channel_->IsLame()) { + channel_->RemoveConnectivityWatcher(watcher_); } // Do an async hop before unreffing. This avoids a deadlock upon // shutdown in the case where the xDS channel is itself an xDS channel @@ -319,11 +304,11 @@ GrpcXdsTransportFactory::GrpcXdsTransport::CreateStreamingCall( return MakeOrphanable( factory_->RefAsSubclass(DEBUG_LOCATION, "StreamingCall"), - channel_, method, std::move(event_handler)); + channel_.get(), method, std::move(event_handler)); } void GrpcXdsTransportFactory::GrpcXdsTransport::ResetBackoff() { - grpc_channel_reset_connect_backoff(channel_); + channel_->ResetConnectionBackoff(); } // diff --git a/src/core/ext/xds/xds_transport_grpc.h b/src/core/ext/xds/xds_transport_grpc.h index a4d25ff75c9..1e80a996cd8 100644 --- a/src/core/ext/xds/xds_transport_grpc.h +++ b/src/core/ext/xds/xds_transport_grpc.h @@ -37,6 +37,7 @@ #include "src/core/lib/iomgr/closure.h" #include "src/core/lib/iomgr/error.h" #include "src/core/lib/iomgr/iomgr_fwd.h" +#include "src/core/lib/surface/channel.h" namespace grpc_core { @@ -70,7 +71,6 @@ class GrpcXdsTransportFactory::GrpcXdsTransport const XdsBootstrap::XdsServer& server, std::function on_connectivity_failure, absl::Status* status); - ~GrpcXdsTransport() override; void Orphan() override; @@ -84,7 +84,7 @@ class GrpcXdsTransportFactory::GrpcXdsTransport class StateWatcher; GrpcXdsTransportFactory* factory_; // Not owned. - grpc_channel* channel_; + OrphanablePtr channel_; StateWatcher* watcher_; }; @@ -92,7 +92,7 @@ class GrpcXdsTransportFactory::GrpcXdsTransport::GrpcStreamingCall : public XdsTransportFactory::XdsTransport::StreamingCall { public: GrpcStreamingCall(RefCountedPtr factory, - grpc_channel* channel, const char* method, + Channel* channel, const char* method, std::unique_ptr event_handler); ~GrpcStreamingCall() override; diff --git a/src/core/lib/channel/call_finalization.h b/src/core/lib/channel/call_finalization.h index 793586f0e3c..a3076677826 100644 --- a/src/core/lib/channel/call_finalization.h +++ b/src/core/lib/channel/call_finalization.h @@ -19,9 +19,9 @@ #include -#include "src/core/lib/channel/channel_stack.h" #include "src/core/lib/promise/context.h" #include "src/core/lib/resource_quota/arena.h" +#include "src/core/lib/transport/call_final_info.h" namespace grpc_core { diff --git a/src/core/lib/channel/call_tracer.h b/src/core/lib/channel/call_tracer.h index 67191d1bc8a..f189f311963 100644 --- a/src/core/lib/channel/call_tracer.h +++ b/src/core/lib/channel/call_tracer.h @@ -36,8 +36,8 @@ #include "src/core/lib/iomgr/error.h" #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/slice/slice_buffer.h" +#include "src/core/lib/transport/call_final_info.h" #include "src/core/lib/transport/metadata_batch.h" -#include "src/core/lib/transport/transport.h" namespace grpc_core { diff --git a/src/core/lib/channel/channelz.h b/src/core/lib/channel/channelz.h index 1713171dd6b..42eb29ef991 100644 --- a/src/core/lib/channel/channelz.h +++ b/src/core/lib/channel/channelz.h @@ -191,6 +191,9 @@ class ChannelNode : public BaseNode { static absl::string_view ChannelArgName() { return GRPC_ARG_CHANNELZ_CHANNEL_NODE; } + static int ChannelArgsCompare(const ChannelNode* a, const ChannelNode* b) { + return QsortCompare(a, b); + } // Returns the string description of the given connectivity state. static const char* GetChannelConnectivityStateChangeString( diff --git a/src/core/lib/compression/compression_internal.cc b/src/core/lib/compression/compression_internal.cc index 484b101b79e..7afb9ac70bf 100644 --- a/src/core/lib/compression/compression_internal.cc +++ b/src/core/lib/compression/compression_internal.cc @@ -29,6 +29,7 @@ #include "absl/strings/str_format.h" #include "absl/strings/str_split.h" +#include #include #include "src/core/lib/channel/channel_args.h" @@ -239,4 +240,36 @@ DefaultCompressionAlgorithmFromChannelArgs(const ChannelArgs& args) { return absl::nullopt; } +grpc_compression_options CompressionOptionsFromChannelArgs( + const ChannelArgs& args) { + // Set compression options. + grpc_compression_options compression_options; + grpc_compression_options_init(&compression_options); + auto default_level = args.GetInt(GRPC_COMPRESSION_CHANNEL_DEFAULT_LEVEL); + if (default_level.has_value()) { + compression_options.default_level.is_set = true; + compression_options.default_level.level = Clamp( + static_cast(*default_level), + GRPC_COMPRESS_LEVEL_NONE, + static_cast(GRPC_COMPRESS_LEVEL_COUNT - 1)); + } + auto default_algorithm = + args.GetInt(GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM); + if (default_algorithm.has_value()) { + compression_options.default_algorithm.is_set = true; + compression_options.default_algorithm.algorithm = + Clamp(static_cast(*default_algorithm), + GRPC_COMPRESS_NONE, + static_cast( + GRPC_COMPRESS_ALGORITHMS_COUNT - 1)); + } + auto enabled_algorithms_bitset = + args.GetInt(GRPC_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET); + if (enabled_algorithms_bitset.has_value()) { + compression_options.enabled_algorithms_bitset = + *enabled_algorithms_bitset | 1 /* always support no compression */; + } + return compression_options; +} + } // namespace grpc_core diff --git a/src/core/lib/compression/compression_internal.h b/src/core/lib/compression/compression_internal.h index 4f92f73ee06..ccbdd4aa9d9 100644 --- a/src/core/lib/compression/compression_internal.h +++ b/src/core/lib/compression/compression_internal.h @@ -88,6 +88,9 @@ class CompressionAlgorithmSet { BitSet set_; }; +grpc_compression_options CompressionOptionsFromChannelArgs( + const ChannelArgs& args); + } // namespace grpc_core #endif // GRPC_SRC_CORE_LIB_COMPRESSION_COMPRESSION_INTERNAL_H diff --git a/src/core/lib/surface/builtins.h b/src/core/lib/surface/builtins.h deleted file mode 100644 index f027cc364e9..00000000000 --- a/src/core/lib/surface/builtins.h +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2021 gRPC authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef GRPC_SRC_CORE_LIB_SURFACE_BUILTINS_H -#define GRPC_SRC_CORE_LIB_SURFACE_BUILTINS_H - -#include - -#include "src/core/lib/config/core_configuration.h" - -namespace grpc_core { -void RegisterBuiltins(CoreConfiguration::Builder* builder); -} // namespace grpc_core - -#endif // GRPC_SRC_CORE_LIB_SURFACE_BUILTINS_H diff --git a/src/core/lib/surface/call.cc b/src/core/lib/surface/call.cc index b4e66974aaf..3f97df0f81f 100644 --- a/src/core/lib/surface/call.cc +++ b/src/core/lib/surface/call.cc @@ -98,7 +98,7 @@ #include "src/core/lib/surface/call_test_only.h" #include "src/core/lib/surface/channel.h" #include "src/core/lib/surface/completion_queue.h" -#include "src/core/lib/surface/server.h" +#include "src/core/lib/surface/server_interface.h" #include "src/core/lib/surface/validate_metadata.h" #include "src/core/lib/surface/wait_for_cq_end_op.h" #include "src/core/lib/transport/batch_builder.h" @@ -757,7 +757,7 @@ class FilterStackCall final : public Call { struct { int* cancelled; // backpointer to owning server if this is a server side call. - Server* core_server; + ServerInterface* core_server; } server; } final_op_; AtomicError status_error_; @@ -3368,7 +3368,7 @@ class ServerPromiseBasedCall final : public PromiseBasedCall, const Completion& completion); void Finish(ServerMetadataHandle result); - Server* const server_; + ServerInterface* const server_; const void* const server_transport_data_; PipeSender* server_initial_metadata_ = nullptr; PipeSender* server_to_client_messages_ = nullptr; @@ -3684,7 +3684,7 @@ class ServerCallSpine final : public CallSpineInterface, public ServerCallContext, public BasicPromiseBasedCall { public: - ServerCallSpine(Server* server, Channel* channel, Arena* arena); + ServerCallSpine(ServerInterface* server, Channel* channel, Arena* arena); // CallSpineInterface Pipe& client_initial_metadata() override { @@ -3771,7 +3771,8 @@ class ServerCallSpine final : public CallSpineInterface, ClientMetadataHandle client_initial_metadata_stored_; }; -ServerCallSpine::ServerCallSpine(Server* server, Channel* channel, Arena* arena) +ServerCallSpine::ServerCallSpine(ServerInterface* server, Channel* channel, + Arena* arena) : BasicPromiseBasedCall(arena, 0, 1, [channel, server]() -> grpc_call_create_args { grpc_call_create_args args; @@ -4119,14 +4120,15 @@ void ServerCallSpine::CommitBatch(const grpc_op* ops, size_t nops, } } -RefCountedPtr MakeServerCall(Server* server, +RefCountedPtr MakeServerCall(ServerInterface* server, Channel* channel, Arena* arena) { return RefCountedPtr( arena->New(server, channel, arena)); } #else -RefCountedPtr MakeServerCall(Server*, Channel*, Arena*) { +RefCountedPtr MakeServerCall(ServerInterface*, Channel*, + Arena*) { Crash("not implemented"); } #endif diff --git a/src/core/lib/surface/call.h b/src/core/lib/surface/call.h index 520cf13505c..1736c83fd92 100644 --- a/src/core/lib/surface/call.h +++ b/src/core/lib/surface/call.h @@ -50,7 +50,7 @@ #include "src/core/lib/slice/slice.h" #include "src/core/lib/surface/api_trace.h" #include "src/core/lib/surface/channel.h" -#include "src/core/lib/surface/server.h" +#include "src/core/lib/surface/server_interface.h" #include "src/core/lib/transport/transport.h" typedef void (*grpc_ioreq_completion_func)(grpc_call* call, int success, @@ -58,7 +58,7 @@ typedef void (*grpc_ioreq_completion_func)(grpc_call* call, int success, typedef struct grpc_call_create_args { grpc_core::RefCountedPtr channel; - grpc_core::Server* server; + grpc_core::ServerInterface* server; grpc_call* parent; uint32_t propagation_mask; @@ -159,7 +159,7 @@ class CallContext { template <> struct ContextType {}; -RefCountedPtr MakeServerCall(Server* server, +RefCountedPtr MakeServerCall(ServerInterface* server, Channel* channel, Arena* arena); diff --git a/src/core/lib/surface/channel.cc b/src/core/lib/surface/channel.cc index 0c6f1ef460a..35aa13bd247 100644 --- a/src/core/lib/surface/channel.cc +++ b/src/core/lib/surface/channel.cc @@ -1,5 +1,4 @@ // -// // Copyright 2015 gRPC authors. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,294 +13,84 @@ // See the License for the specific language governing permissions and // limitations under the License. // -// #include #include "src/core/lib/surface/channel.h" -#include -#include - -#include -#include -#include -#include - -#include "absl/status/status.h" - #include #include #include #include #include -#include #include "src/core/lib/channel/channel_args.h" -#include "src/core/lib/channel/channel_stack.h" -#include "src/core/lib/channel/channel_stack_builder_impl.h" #include "src/core/lib/channel/channel_trace.h" #include "src/core/lib/channel/channelz.h" -#include "src/core/lib/channel/metrics.h" -#include "src/core/lib/config/core_configuration.h" +#include "src/core/lib/compression/compression_internal.h" #include "src/core/lib/debug/stats.h" #include "src/core/lib/debug/stats_data.h" #include "src/core/lib/debug/trace.h" -#include "src/core/lib/gpr/useful.h" -#include "src/core/lib/gprpp/manual_constructor.h" -#include "src/core/lib/gprpp/ref_counted_ptr.h" -#include "src/core/lib/iomgr/error.h" #include "src/core/lib/iomgr/exec_ctx.h" -#include "src/core/lib/resource_quota/memory_quota.h" -#include "src/core/lib/resource_quota/resource_quota.h" #include "src/core/lib/surface/api_trace.h" -#include "src/core/lib/surface/call.h" -#include "src/core/lib/surface/channel_init.h" -#include "src/core/lib/surface/channel_stack_type.h" -#include "src/core/lib/surface/init_internally.h" -#include "src/core/lib/transport/call_factory.h" -#include "src/core/lib/transport/transport.h" - -// IWYU pragma: no_include namespace grpc_core { -namespace { +// +// Channel::RegisteredCall +// -class NotReallyACallFactory final : public CallFactory { - public: - using CallFactory::CallFactory; - CallInitiator CreateCall(ClientMetadataHandle, Arena*) override { - Crash("NotReallyACallFactory::CreateCall should never be called"); +Channel::RegisteredCall::RegisteredCall(const char* method_arg, + const char* host_arg) { + path = Slice::FromCopiedString(method_arg); + if (host_arg != nullptr && host_arg[0] != 0) { + authority = Slice::FromCopiedString(host_arg); } -}; - -} // namespace - -Channel::Channel(bool is_client, bool is_promising, std::string target, - const ChannelArgs& channel_args, - grpc_compression_options compression_options, - RefCountedPtr channel_stack) - : is_client_(is_client), - is_promising_(is_promising), - compression_options_(compression_options), - channelz_node_(channel_args.GetObjectRef()), - target_(std::move(target)), - channel_stack_(std::move(channel_stack)), - call_factory_(MakeRefCounted(channel_args)) { - // We need to make sure that grpc_shutdown() does not shut things down - // until after the channel is destroyed. However, the channel may not - // actually be destroyed by the time grpc_channel_destroy() returns, - // since there may be other existing refs to the channel. If those - // refs are held by things that are visible to the wrapped language - // (such as outstanding calls on the channel), then the wrapped - // language can be responsible for making sure that grpc_shutdown() - // does not run until after those refs are released. However, the - // channel may also have refs to itself held internally for various - // things that need to be cleaned up at channel destruction (e.g., - // LB policies, subchannels, etc), and because these refs are not - // visible to the wrapped language, it cannot be responsible for - // deferring grpc_shutdown() until after they are released. To - // accommodate that, we call grpc_init() here and then call - // grpc_shutdown() when the channel is actually destroyed, thus - // ensuring that shutdown is deferred until that point. - InitInternally(); - auto channelz_node = channelz_node_; - *channel_stack_->on_destroy = [channelz_node]() { - if (channelz_node != nullptr) { - channelz_node->AddTraceEvent( - channelz::ChannelTrace::Severity::Info, - grpc_slice_from_static_string("Channel destroyed")); - } - ShutdownInternally(); - }; } -absl::StatusOr> Channel::CreateWithBuilder( - ChannelStackBuilder* builder) { - auto channel_args = builder->channel_args(); - if (builder->channel_stack_type() == GRPC_SERVER_CHANNEL) { - global_stats().IncrementServerChannelsCreated(); - } else { - global_stats().IncrementClientChannelsCreated(); - } - absl::StatusOr> r = builder->Build(); - if (!r.ok()) { - auto status = r.status(); - gpr_log(GPR_ERROR, "channel stack builder failed: %s", - status.ToString().c_str()); - return status; - } - - grpc_compression_options compression_options; - grpc_compression_options_init(&compression_options); - auto default_level = - channel_args.GetInt(GRPC_COMPRESSION_CHANNEL_DEFAULT_LEVEL); - if (default_level.has_value()) { - compression_options.default_level.is_set = true; - compression_options.default_level.level = Clamp( - static_cast(*default_level), - GRPC_COMPRESS_LEVEL_NONE, - static_cast(GRPC_COMPRESS_LEVEL_COUNT - 1)); - } - auto default_algorithm = - channel_args.GetInt(GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM); - if (default_algorithm.has_value()) { - compression_options.default_algorithm.is_set = true; - compression_options.default_algorithm.algorithm = - Clamp(static_cast(*default_algorithm), - GRPC_COMPRESS_NONE, - static_cast( - GRPC_COMPRESS_ALGORITHMS_COUNT - 1)); - } - auto enabled_algorithms_bitset = - channel_args.GetInt(GRPC_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET); - if (enabled_algorithms_bitset.has_value()) { - compression_options.enabled_algorithms_bitset = - *enabled_algorithms_bitset | 1 /* always support no compression */; +Channel::RegisteredCall::RegisteredCall(const RegisteredCall& other) + : path(other.path.Ref()) { + if (other.authority.has_value()) { + authority = other.authority->Ref(); } - - // TODO(roth): Populate authority after merging - // https://github.com/grpc/grpc/pull/35924. - StatsPlugin::ChannelScope scope(builder->target(), ""); - *(*r)->stats_plugin_group = - GlobalStatsPluginRegistry::GetStatsPluginsForChannel(scope); - - return RefCountedPtr(new Channel( - grpc_channel_stack_type_is_client(builder->channel_stack_type()), - builder->IsPromising(), std::string(builder->target()), channel_args, - compression_options, std::move(*r))); } -namespace { +Channel::RegisteredCall::~RegisteredCall() {} -void* channelz_node_copy(void* p) { - channelz::ChannelNode* node = static_cast(p); - node->Ref().release(); - return p; -} -void channelz_node_destroy(void* p) { - channelz::ChannelNode* node = static_cast(p); - node->Unref(); -} -int channelz_node_cmp(void* p1, void* p2) { return QsortCompare(p1, p2); } -const grpc_arg_pointer_vtable channelz_node_arg_vtable = { - channelz_node_copy, channelz_node_destroy, channelz_node_cmp}; -} // namespace +// +// Channel +// -absl::StatusOr> Channel::Create( - const char* target, ChannelArgs args, - grpc_channel_stack_type channel_stack_type, Transport* optional_transport) { - if (!args.GetString(GRPC_ARG_DEFAULT_AUTHORITY).has_value()) { - auto ssl_override = args.GetString(GRPC_SSL_TARGET_NAME_OVERRIDE_ARG); - if (ssl_override.has_value()) { - args = args.Set(GRPC_ARG_DEFAULT_AUTHORITY, - std::string(ssl_override.value())); - } - } - if (grpc_channel_stack_type_is_client(channel_stack_type)) { - auto channel_args_mutator = - grpc_channel_args_get_client_channel_creation_mutator(); - if (channel_args_mutator != nullptr) { - args = channel_args_mutator(target, args, channel_stack_type); - } - } - // We only need to do this for clients here. For servers, this will be - // done in src/core/lib/surface/server.cc. - if (grpc_channel_stack_type_is_client(channel_stack_type)) { - // Check whether channelz is enabled. - if (args.GetBool(GRPC_ARG_ENABLE_CHANNELZ) - .value_or(GRPC_ENABLE_CHANNELZ_DEFAULT)) { - // Get parameters needed to create the channelz node. - const size_t channel_tracer_max_memory = std::max( - 0, - args.GetInt(GRPC_ARG_MAX_CHANNEL_TRACE_EVENT_MEMORY_PER_NODE) - .value_or(GRPC_MAX_CHANNEL_TRACE_EVENT_MEMORY_PER_NODE_DEFAULT)); - const bool is_internal_channel = - args.GetBool(GRPC_ARG_CHANNELZ_IS_INTERNAL_CHANNEL).value_or(false); - // Create the channelz node. - std::string channelz_node_target{target == nullptr ? "unknown" : target}; - RefCountedPtr channelz_node = - MakeRefCounted(channelz_node_target, - channel_tracer_max_memory, - is_internal_channel); - channelz_node->AddTraceEvent( - channelz::ChannelTrace::Severity::Info, - grpc_slice_from_static_string("Channel created")); - // Add channelz node to channel args. - // We remove the is_internal_channel arg, since we no longer need it. - args = args.Remove(GRPC_ARG_CHANNELZ_IS_INTERNAL_CHANNEL) - .Set(GRPC_ARG_CHANNELZ_CHANNEL_NODE, - ChannelArgs::Pointer(channelz_node.release(), - &channelz_node_arg_vtable)); - } - } - ChannelStackBuilderImpl builder( - grpc_channel_stack_type_string(channel_stack_type), channel_stack_type, - args.SetObject(optional_transport)); - builder.SetTarget(target); - if (!CoreConfiguration::Get().channel_init().CreateStack(&builder)) { - return nullptr; +Channel::Channel(std::string target, const ChannelArgs& channel_args) + : target_(std::move(target)), + channelz_node_(channel_args.GetObjectRef()), + compression_options_(CompressionOptionsFromChannelArgs(channel_args)) {} + +Channel::RegisteredCall* Channel::RegisterCall(const char* method, + const char* host) { + MutexLock lock(&mu_); + auto key = std::make_pair(std::string(host != nullptr ? host : ""), + std::string(method != nullptr ? method : "")); + auto rc_posn = registration_table_.find(key); + if (rc_posn != registration_table_.end()) { + return &rc_posn->second; } - return CreateWithBuilder(&builder); + auto insertion_result = registration_table_.insert( + {std::move(key), RegisteredCall(method, host)}); + return &insertion_result.first->second; } } // namespace grpc_core -char* grpc_channel_get_target(grpc_channel* channel) { - GRPC_API_TRACE("grpc_channel_get_target(channel=%p)", 1, (channel)); - auto target = grpc_core::Channel::FromC(channel)->target(); - char* buffer = static_cast(gpr_zalloc(target.size() + 1)); - memcpy(buffer, target.data(), target.size()); - return buffer; -} - -void grpc_channel_get_info(grpc_channel* channel, - const grpc_channel_info* channel_info) { - grpc_core::ApplicationCallbackExecCtx callback_exec_ctx; - grpc_core::ExecCtx exec_ctx; - grpc_channel_element* elem = grpc_channel_stack_element( - grpc_core::Channel::FromC(channel)->channel_stack(), 0); - elem->filter->get_channel_info(elem, channel_info); -} +// +// C-core API +// -void grpc_channel_reset_connect_backoff(grpc_channel* channel) { +void grpc_channel_destroy(grpc_channel* channel) { grpc_core::ApplicationCallbackExecCtx callback_exec_ctx; grpc_core::ExecCtx exec_ctx; - GRPC_API_TRACE("grpc_channel_reset_connect_backoff(channel=%p)", 1, - (channel)); - grpc_transport_op* op = grpc_make_transport_op(nullptr); - op->reset_connect_backoff = true; - grpc_channel_element* elem = grpc_channel_stack_element( - grpc_core::Channel::FromC(channel)->channel_stack(), 0); - elem->filter->start_transport_op(elem, op); -} - -static grpc_call* grpc_channel_create_call_internal( - grpc_channel* c_channel, grpc_call* parent_call, uint32_t propagation_mask, - grpc_completion_queue* cq, grpc_pollset_set* pollset_set_alternative, - grpc_core::Slice path, absl::optional authority, - grpc_core::Timestamp deadline, bool registered_method) { - auto channel = grpc_core::Channel::FromC(c_channel)->Ref(); - GPR_ASSERT(channel->is_client()); - GPR_ASSERT(!(cq != nullptr && pollset_set_alternative != nullptr)); - - grpc_call_create_args args; - args.channel = std::move(channel); - args.server = nullptr; - args.parent = parent_call; - args.propagation_mask = propagation_mask; - args.cq = cq; - args.pollset_set_alternative = pollset_set_alternative; - args.server_transport_data = nullptr; - args.path = std::move(path); - args.authority = std::move(authority); - args.send_deadline = deadline; - args.registered_method = registered_method; - - grpc_call* call; - GRPC_LOG_IF_ERROR("call_create", grpc_call_create(&args, &call)); - return call; + GRPC_API_TRACE("grpc_channel_destroy(channel=%p)", 1, (channel)); + grpc_channel_destroy_internal(channel); } grpc_call* grpc_channel_create_call(grpc_channel* channel, @@ -313,52 +102,16 @@ grpc_call* grpc_channel_create_call(grpc_channel* channel, GPR_ASSERT(!reserved); grpc_core::ApplicationCallbackExecCtx callback_exec_ctx; grpc_core::ExecCtx exec_ctx; - grpc_call* call = grpc_channel_create_call_internal( - channel, parent_call, propagation_mask, completion_queue, nullptr, + return grpc_core::Channel::FromC(channel)->CreateCall( + parent_call, propagation_mask, completion_queue, nullptr, grpc_core::Slice(grpc_core::CSliceRef(method)), host != nullptr ? absl::optional(grpc_core::CSliceRef(*host)) : absl::nullopt, grpc_core::Timestamp::FromTimespecRoundUp(deadline), /*registered_method=*/false); - - return call; -} - -grpc_call* grpc_channel_create_pollset_set_call( - grpc_channel* channel, grpc_call* parent_call, uint32_t propagation_mask, - grpc_pollset_set* pollset_set, const grpc_slice& method, - const grpc_slice* host, grpc_core::Timestamp deadline, void* reserved) { - GPR_ASSERT(!reserved); - return grpc_channel_create_call_internal( - channel, parent_call, propagation_mask, nullptr, pollset_set, - grpc_core::Slice(method), - host != nullptr - ? absl::optional(grpc_core::CSliceRef(*host)) - : absl::nullopt, - deadline, /*registered_method=*/true); } -namespace grpc_core { - -RegisteredCall::RegisteredCall(const char* method_arg, const char* host_arg) { - path = Slice::FromCopiedString(method_arg); - if (host_arg != nullptr && host_arg[0] != 0) { - authority = Slice::FromCopiedString(host_arg); - } -} - -RegisteredCall::RegisteredCall(const RegisteredCall& other) - : path(other.path.Ref()) { - if (other.authority.has_value()) { - authority = other.authority->Ref(); - } -} - -RegisteredCall::~RegisteredCall() {} - -} // namespace grpc_core - void* grpc_channel_register_call(grpc_channel* channel, const char* method, const char* host, void* reserved) { GRPC_API_TRACE( @@ -370,29 +123,12 @@ void* grpc_channel_register_call(grpc_channel* channel, const char* method, return grpc_core::Channel::FromC(channel)->RegisterCall(method, host); } -namespace grpc_core { - -RegisteredCall* Channel::RegisterCall(const char* method, const char* host) { - MutexLock lock(®istration_table_.mu); - auto key = std::make_pair(std::string(host != nullptr ? host : ""), - std::string(method != nullptr ? method : "")); - auto rc_posn = registration_table_.map.find(key); - if (rc_posn != registration_table_.map.end()) { - return &rc_posn->second; - } - auto insertion_result = registration_table_.map.insert( - {std::move(key), RegisteredCall(method, host)}); - return &insertion_result.first->second; -} - -} // namespace grpc_core - grpc_call* grpc_channel_create_registered_call( grpc_channel* channel, grpc_call* parent_call, uint32_t propagation_mask, grpc_completion_queue* completion_queue, void* registered_call_handle, gpr_timespec deadline, void* reserved) { - grpc_core::RegisteredCall* rc = - static_cast(registered_call_handle); + auto* rc = + static_cast(registered_call_handle); GRPC_API_TRACE( "grpc_channel_create_registered_call(" "channel=%p, parent_call=%p, propagation_mask=%x, completion_queue=%p, " @@ -407,31 +143,77 @@ grpc_call* grpc_channel_create_registered_call( GPR_ASSERT(!reserved); grpc_core::ApplicationCallbackExecCtx callback_exec_ctx; grpc_core::ExecCtx exec_ctx; - grpc_call* call = grpc_channel_create_call_internal( - channel, parent_call, propagation_mask, completion_queue, nullptr, - rc->path.Ref(), + return grpc_core::Channel::FromC(channel)->CreateCall( + parent_call, propagation_mask, completion_queue, nullptr, rc->path.Ref(), rc->authority.has_value() ? absl::optional(rc->authority->Ref()) : absl::nullopt, grpc_core::Timestamp::FromTimespecRoundUp(deadline), /*registered_method=*/true); +} - return call; +char* grpc_channel_get_target(grpc_channel* channel) { + GRPC_API_TRACE("grpc_channel_get_target(channel=%p)", 1, (channel)); + auto target = grpc_core::Channel::FromC(channel)->target(); + char* buffer = static_cast(gpr_zalloc(target.size() + 1)); + memcpy(buffer, target.data(), target.size()); + return buffer; } -void grpc_channel_destroy_internal(grpc_channel* c_channel) { - grpc_core::RefCountedPtr channel( - grpc_core::Channel::FromC(c_channel)); - grpc_transport_op* op = grpc_make_transport_op(nullptr); - grpc_channel_element* elem; - GRPC_API_TRACE("grpc_channel_destroy(channel=%p)", 1, (c_channel)); - op->disconnect_with_error = GRPC_ERROR_CREATE("Channel Destroyed"); - elem = grpc_channel_stack_element(channel->channel_stack(), 0); - elem->filter->start_transport_op(elem, op); +void grpc_channel_get_info(grpc_channel* channel, + const grpc_channel_info* channel_info) { + grpc_core::ApplicationCallbackExecCtx callback_exec_ctx; + grpc_core::ExecCtx exec_ctx; + grpc_core::Channel::FromC(channel)->GetInfo(channel_info); } -void grpc_channel_destroy(grpc_channel* channel) { +void grpc_channel_reset_connect_backoff(grpc_channel* channel) { grpc_core::ApplicationCallbackExecCtx callback_exec_ctx; grpc_core::ExecCtx exec_ctx; - grpc_channel_destroy_internal(channel); + GRPC_API_TRACE("grpc_channel_reset_connect_backoff(channel=%p)", 1, + (channel)); + grpc_core::Channel::FromC(channel)->ResetConnectionBackoff(); +} + +int grpc_channel_support_connectivity_watcher(grpc_channel* channel) { + return grpc_core::Channel::FromC(channel)->SupportsConnectivityWatcher(); +} + +grpc_connectivity_state grpc_channel_check_connectivity_state( + grpc_channel* channel, int try_to_connect) { + grpc_core::ApplicationCallbackExecCtx callback_exec_ctx; + grpc_core::ExecCtx exec_ctx; + GRPC_API_TRACE( + "grpc_channel_check_connectivity_state(channel=%p, try_to_connect=%d)", 2, + (channel, try_to_connect)); + return grpc_core::Channel::FromC(channel)->CheckConnectivityState( + try_to_connect); +} + +void grpc_channel_watch_connectivity_state( + grpc_channel* channel, grpc_connectivity_state last_observed_state, + gpr_timespec deadline, grpc_completion_queue* cq, void* tag) { + grpc_core::ApplicationCallbackExecCtx callback_exec_ctx; + grpc_core::ExecCtx exec_ctx; + GRPC_API_TRACE( + "grpc_channel_watch_connectivity_state(" + "channel=%p, last_observed_state=%d, " + "deadline=gpr_timespec { tv_sec: %" PRId64 + ", tv_nsec: %d, clock_type: %d }, " + "cq=%p, tag=%p)", + 7, + (channel, (int)last_observed_state, deadline.tv_sec, deadline.tv_nsec, + (int)deadline.clock_type, cq, tag)); + return grpc_core::Channel::FromC(channel)->WatchConnectivityState( + last_observed_state, grpc_core::Timestamp::FromTimespecRoundUp(deadline), + cq, tag); +} + +void grpc_channel_ping(grpc_channel* channel, grpc_completion_queue* cq, + void* tag, void* reserved) { + grpc_core::ExecCtx exec_ctx; + GRPC_API_TRACE("grpc_channel_ping(channel=%p, cq=%p, tag=%p, reserved=%p)", 4, + (channel, cq, tag, reserved)); + GPR_ASSERT(reserved == nullptr); + grpc_core::Channel::FromC(channel)->Ping(cq, tag); } diff --git a/src/core/lib/surface/channel.h b/src/core/lib/surface/channel.h index 4796331893e..d41282444a3 100644 --- a/src/core/lib/surface/channel.h +++ b/src/core/lib/surface/channel.h @@ -1,6 +1,5 @@ // -// -// Copyright 2015 gRPC authors. +// Copyright 2024 gRPC authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,20 +13,14 @@ // See the License for the specific language governing permissions and // limitations under the License. // -// #ifndef GRPC_SRC_CORE_LIB_SURFACE_CHANNEL_H #define GRPC_SRC_CORE_LIB_SURFACE_CHANNEL_H #include -#include -#include - -#include #include #include -#include #include "absl/base/thread_annotations.h" #include "absl/status/statusor.h" @@ -35,155 +28,143 @@ #include "absl/types/optional.h" #include -#include #include #include -#include +#include #include "src/core/lib/channel/channel_args.h" -#include "src/core/lib/channel/channel_fwd.h" -#include "src/core/lib/channel/channel_stack.h" // IWYU pragma: keep -#include "src/core/lib/channel/channel_stack_builder.h" #include "src/core/lib/channel/channelz.h" #include "src/core/lib/gprpp/cpp_impl_of.h" -#include "src/core/lib/gprpp/debug_location.h" #include "src/core/lib/gprpp/ref_counted.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/gprpp/sync.h" #include "src/core/lib/gprpp/time.h" #include "src/core/lib/iomgr/iomgr_fwd.h" +#include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/slice/slice.h" #include "src/core/lib/surface/channel_stack_type.h" -#include "src/core/lib/transport/call_factory.h" -#include "src/core/lib/transport/transport.h" +#include "src/core/lib/transport/connectivity_state.h" -/// The same as grpc_channel_destroy, but doesn't create an ExecCtx, and so -/// is safe to use from within core. -void grpc_channel_destroy_internal(grpc_channel* channel); - -/// Create a call given a grpc_channel, in order to call \a method. -/// Progress is tied to activity on \a pollset_set. The returned call object is -/// meant to be used with \a grpc_call_start_batch_and_execute, which relies on -/// callbacks to signal completions. \a method and \a host need -/// only live through the invocation of this function. If \a parent_call is -/// non-NULL, it must be a server-side call. It will be used to propagate -/// properties from the server call to this new client call, depending on the -/// value of \a propagation_mask (see propagation_bits.h for possible values) -grpc_call* grpc_channel_create_pollset_set_call( - grpc_channel* channel, grpc_call* parent_call, uint32_t propagation_mask, - grpc_pollset_set* pollset_set, const grpc_slice& method, - const grpc_slice* host, grpc_core::Timestamp deadline, void* reserved); - -/// Get a (borrowed) pointer to this channels underlying channel stack -grpc_channel_stack* grpc_channel_get_channel_stack(grpc_channel* channel); - -grpc_core::channelz::ChannelNode* grpc_channel_get_channelz_node( - grpc_channel* channel); +// Forward declaration to avoid dependency loop. +struct grpc_channel_stack; namespace grpc_core { -struct RegisteredCall { - Slice path; - absl::optional authority; - - explicit RegisteredCall(const char* method_arg, const char* host_arg); - RegisteredCall(const RegisteredCall& other); - RegisteredCall& operator=(const RegisteredCall&) = delete; - - ~RegisteredCall(); -}; - -struct CallRegistrationTable { - Mutex mu; - // The map key should be owned strings rather than unowned char*'s to - // guarantee that it outlives calls on the core channel (which may outlast the - // C++ or other wrapped language Channel that registered these calls). - std::map, RegisteredCall> map - ABSL_GUARDED_BY(mu); -}; +// Forward declaration to avoid dependency loop. +class Transport; class Channel : public RefCounted, public CppImplOf { public: - static absl::StatusOr> Create( - const char* target, ChannelArgs args, - grpc_channel_stack_type channel_stack_type, - Transport* optional_transport); + struct RegisteredCall { + Slice path; + absl::optional authority; - static absl::StatusOr> CreateWithBuilder( - ChannelStackBuilder* builder); + explicit RegisteredCall(const char* method_arg, const char* host_arg); + RegisteredCall(const RegisteredCall& other); + RegisteredCall& operator=(const RegisteredCall&) = delete; - grpc_channel_stack* channel_stack() const { return channel_stack_.get(); } + ~RegisteredCall(); + }; - grpc_compression_options compression_options() const { - return compression_options_; - } + virtual void Orphan() = 0; - channelz::ChannelNode* channelz_node() const { return channelz_node_.get(); } + virtual Arena* CreateArena() = 0; + virtual void DestroyArena(Arena* arena) = 0; + + virtual bool IsLame() const = 0; + + // TODO(roth): This should return a C++ type. + virtual grpc_call* CreateCall(grpc_call* parent_call, + uint32_t propagation_mask, + grpc_completion_queue* cq, + grpc_pollset_set* pollset_set_alternative, + Slice path, absl::optional authority, + Timestamp deadline, bool registered_method) = 0; + + virtual grpc_event_engine::experimental::EventEngine* event_engine() + const = 0; + + virtual bool SupportsConnectivityWatcher() const = 0; - Arena* CreateArena() { return call_factory_->CreateArena(); } - void DestroyArena(Arena* arena) { return call_factory_->DestroyArena(arena); } + virtual grpc_connectivity_state CheckConnectivityState( + bool try_to_connect) = 0; + + // For external watched via the C-core API. + virtual void WatchConnectivityState( + grpc_connectivity_state last_observed_state, Timestamp deadline, + grpc_completion_queue* cq, void* tag) = 0; + + // For internal watches. + virtual void AddConnectivityWatcher( + grpc_connectivity_state initial_state, + OrphanablePtr watcher) = 0; + virtual void RemoveConnectivityWatcher( + AsyncConnectivityStateWatcherInterface* watcher) = 0; + + virtual void GetInfo(const grpc_channel_info* channel_info) = 0; + + virtual void ResetConnectionBackoff() = 0; absl::string_view target() const { return target_; } - bool is_client() const { return is_client_; } - bool is_promising() const { return is_promising_; } + channelz::ChannelNode* channelz_node() const { return channelz_node_.get(); } + grpc_compression_options compression_options() const { + return compression_options_; + } + RegisteredCall* RegisterCall(const char* method, const char* host); int TestOnlyRegisteredCalls() { - MutexLock lock(®istration_table_.mu); - return registration_table_.map.size(); + MutexLock lock(&mu_); + return registration_table_.size(); } - grpc_event_engine::experimental::EventEngine* event_engine() const { - return channel_stack_->EventEngine(); - } + // For tests only. + // Pings the channel's peer. Load-balanced channels will select one + // subchannel to ping. If the channel is not connected, posts a + // failure to the CQ. + virtual void Ping(grpc_completion_queue* cq, void* tag) = 0; - private: - Channel(bool is_client, bool is_promising, std::string target, - const ChannelArgs& channel_args, - grpc_compression_options compression_options, - RefCountedPtr channel_stack); + // TODO(roth): Remove these methods when LegacyChannel goes away. + virtual grpc_channel_stack* channel_stack() const { return nullptr; } + virtual bool is_client() const { return true; } + virtual bool is_promising() const { return true; } - const bool is_client_; - const bool is_promising_; + protected: + Channel(std::string target, const ChannelArgs& channel_args); + + private: + const std::string target_; + const RefCountedPtr channelz_node_; const grpc_compression_options compression_options_; - CallRegistrationTable registration_table_; - RefCountedPtr channelz_node_; - std::string target_; - const RefCountedPtr channel_stack_; - const RefCountedPtr call_factory_; + + Mutex mu_; + // The map key needs to be owned strings rather than unowned char*'s to + // guarantee that it outlives calls on the core channel (which may outlast + // the C++ or other wrapped language Channel that registered these calls). + std::map, RegisteredCall> + registration_table_ ABSL_GUARDED_BY(mu_); }; } // namespace grpc_core +/// The same as grpc_channel_destroy, but doesn't create an ExecCtx, and so +/// is safe to use from within core. +inline void grpc_channel_destroy_internal(grpc_channel* channel) { + grpc_core::Channel::FromC(channel)->Orphan(); +} + +// Return the channel's compression options. inline grpc_compression_options grpc_channel_compression_options( const grpc_channel* channel) { return grpc_core::Channel::FromC(channel)->compression_options(); } -inline grpc_channel_stack* grpc_channel_get_channel_stack( - grpc_channel* channel) { - return grpc_core::Channel::FromC(channel)->channel_stack(); -} - inline grpc_core::channelz::ChannelNode* grpc_channel_get_channelz_node( grpc_channel* channel) { return grpc_core::Channel::FromC(channel)->channelz_node(); } -inline void grpc_channel_internal_ref(grpc_channel* channel, - const char* reason) { - grpc_core::Channel::FromC(channel)->Ref(DEBUG_LOCATION, reason).release(); -} -inline void grpc_channel_internal_unref(grpc_channel* channel, - const char* reason) { - grpc_core::Channel::FromC(channel)->Unref(DEBUG_LOCATION, reason); -} - -// Return the channel's compression options. -grpc_compression_options grpc_channel_compression_options( - const grpc_channel* channel); - // Ping the channels peer (load balanced channels will select one sub-channel to // ping); if the channel is not connected, posts a failed. void grpc_channel_ping(grpc_channel* channel, grpc_completion_queue* cq, diff --git a/src/core/lib/surface/channel_create.cc b/src/core/lib/surface/channel_create.cc new file mode 100644 index 00000000000..c98da2824b9 --- /dev/null +++ b/src/core/lib/surface/channel_create.cc @@ -0,0 +1,100 @@ +// +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include + +#include +#include +#include + +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/channel/channelz.h" +#include "src/core/lib/debug/stats.h" +#include "src/core/lib/debug/stats_data.h" +#include "src/core/lib/surface/channel.h" +#include "src/core/lib/surface/lame_client.h" +#include "src/core/lib/surface/legacy_channel.h" + +namespace grpc_core { + +absl::StatusOr> ChannelCreate( + std::string target, ChannelArgs args, + grpc_channel_stack_type channel_stack_type, Transport* optional_transport) { + global_stats().IncrementClientChannelsCreated(); + // Set default authority if needed. + if (!args.GetString(GRPC_ARG_DEFAULT_AUTHORITY).has_value()) { + auto ssl_override = args.GetString(GRPC_SSL_TARGET_NAME_OVERRIDE_ARG); + if (ssl_override.has_value()) { + args = args.Set(GRPC_ARG_DEFAULT_AUTHORITY, + std::string(ssl_override.value())); + } + } + // Check whether channelz is enabled. + if (args.GetBool(GRPC_ARG_ENABLE_CHANNELZ) + .value_or(GRPC_ENABLE_CHANNELZ_DEFAULT)) { + // Get parameters needed to create the channelz node. + const size_t channel_tracer_max_memory = std::max( + 0, args.GetInt(GRPC_ARG_MAX_CHANNEL_TRACE_EVENT_MEMORY_PER_NODE) + .value_or(GRPC_MAX_CHANNEL_TRACE_EVENT_MEMORY_PER_NODE_DEFAULT)); + const bool is_internal_channel = + args.GetBool(GRPC_ARG_CHANNELZ_IS_INTERNAL_CHANNEL).value_or(false); + // Create the channelz node. + std::string channelz_node_target{target.empty() ? "unknown" : target}; + auto channelz_node = MakeRefCounted( + channelz_node_target, channel_tracer_max_memory, is_internal_channel); + channelz_node->AddTraceEvent( + channelz::ChannelTrace::Severity::Info, + grpc_slice_from_static_string("Channel created")); + // Add channelz node to channel args. + // We remove the is_internal_channel arg, since we no longer need it. + args = args.Remove(GRPC_ARG_CHANNELZ_IS_INTERNAL_CHANNEL) + .SetObject(std::move(channelz_node)); + } + // Add transport to args. + if (optional_transport != nullptr) { + args = args.SetObject(optional_transport); + } + // Delegate to legacy channel impl. + return LegacyChannel::Create(std::move(target), std::move(args), + channel_stack_type); +} + +} // namespace grpc_core + +grpc_channel* grpc_lame_client_channel_create(const char* target, + grpc_status_code error_code, + const char* error_message) { + grpc_core::ExecCtx exec_ctx; + GRPC_API_TRACE( + "grpc_lame_client_channel_create(target=%s, error_code=%d, " + "error_message=%s)", + 3, (target, (int)error_code, error_message)); + if (error_code == GRPC_STATUS_OK) error_code = GRPC_STATUS_UNKNOWN; + grpc_core::ChannelArgs args = + grpc_core::CoreConfiguration::Get() + .channel_args_preconditioning() + .PreconditionChannelArgs(nullptr) + .Set(GRPC_ARG_LAME_FILTER_ERROR, + grpc_core::ChannelArgs::Pointer( + new absl::Status(static_cast(error_code), + error_message), + &grpc_core::kLameFilterErrorArgVtable)); + auto channel = + grpc_core::ChannelCreate(target == nullptr ? "" : target, std::move(args), + GRPC_CLIENT_LAME_CHANNEL, nullptr); + GPR_ASSERT(channel.ok()); + return channel->release()->c_ptr(); +} diff --git a/src/core/lib/surface/builtins.cc b/src/core/lib/surface/channel_create.h similarity index 53% rename from src/core/lib/surface/builtins.cc rename to src/core/lib/surface/channel_create.h index 303fbb4dadc..7cc386f379a 100644 --- a/src/core/lib/surface/builtins.cc +++ b/src/core/lib/surface/channel_create.h @@ -1,4 +1,5 @@ -// Copyright 2021 gRPC authors. +// +// Copyright 2024 gRPC authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,27 +12,31 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +// + +#ifndef GRPC_SRC_CORE_LIB_SURFACE_CHANNEL_CREATE_H +#define GRPC_SRC_CORE_LIB_SURFACE_CHANNEL_CREATE_H #include -#include "src/core/lib/surface/builtins.h" +#include -#include "src/core/lib/channel/call_tracer.h" -#include "src/core/lib/config/core_configuration.h" +#include "absl/status/statusor.h" + +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/gprpp/orphanable.h" +#include "src/core/lib/surface/channel.h" #include "src/core/lib/surface/channel_stack_type.h" -#include "src/core/lib/surface/lame_client.h" -#include "src/core/lib/surface/server.h" namespace grpc_core { -void RegisterBuiltins(CoreConfiguration::Builder* builder) { - RegisterServerCallTracerFilter(builder); - builder->channel_init() - ->RegisterFilter(GRPC_CLIENT_LAME_CHANNEL) - .Terminal(); - builder->channel_init() - ->RegisterFilter(GRPC_SERVER_CHANNEL, &Server::kServerTopFilter) - .BeforeAll(); -} +class Transport; + +// Creates a client channel. +absl::StatusOr> ChannelCreate( + std::string target, ChannelArgs args, + grpc_channel_stack_type channel_stack_type, Transport* optional_transport); } // namespace grpc_core + +#endif // GRPC_SRC_CORE_LIB_SURFACE_CHANNEL_CREATE_H diff --git a/src/core/lib/surface/channel_ping.cc b/src/core/lib/surface/channel_ping.cc deleted file mode 100644 index d07b8bc0b3c..00000000000 --- a/src/core/lib/surface/channel_ping.cc +++ /dev/null @@ -1,69 +0,0 @@ -// -// -// Copyright 2015 gRPC authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// - -#include - -#include -#include -#include - -#include "src/core/lib/channel/channel_fwd.h" -#include "src/core/lib/channel/channel_stack.h" -#include "src/core/lib/debug/trace.h" -#include "src/core/lib/iomgr/closure.h" -#include "src/core/lib/iomgr/error.h" -#include "src/core/lib/iomgr/exec_ctx.h" -#include "src/core/lib/surface/api_trace.h" -#include "src/core/lib/surface/channel.h" -#include "src/core/lib/surface/completion_queue.h" -#include "src/core/lib/transport/transport.h" - -struct ping_result { - grpc_closure closure; - void* tag; - grpc_completion_queue* cq; - grpc_cq_completion completion_storage; -}; -static void ping_destroy(void* arg, grpc_cq_completion* /*storage*/) { - gpr_free(arg); -} - -static void ping_done(void* arg, grpc_error_handle error) { - ping_result* pr = static_cast(arg); - grpc_cq_end_op(pr->cq, pr->tag, error, ping_destroy, pr, - &pr->completion_storage); -} - -void grpc_channel_ping(grpc_channel* channel, grpc_completion_queue* cq, - void* tag, void* reserved) { - GRPC_API_TRACE("grpc_channel_ping(channel=%p, cq=%p, tag=%p, reserved=%p)", 4, - (channel, cq, tag, reserved)); - grpc_transport_op* op = grpc_make_transport_op(nullptr); - ping_result* pr = static_cast(gpr_malloc(sizeof(*pr))); - grpc_channel_element* top_elem = - grpc_channel_stack_element(grpc_channel_get_channel_stack(channel), 0); - grpc_core::ExecCtx exec_ctx; - GPR_ASSERT(reserved == nullptr); - pr->tag = tag; - pr->cq = cq; - GRPC_CLOSURE_INIT(&pr->closure, ping_done, pr, grpc_schedule_on_exec_ctx); - op->send_ping.on_ack = &pr->closure; - op->bind_pollset = grpc_cq_pollset(cq); - GPR_ASSERT(grpc_cq_begin_op(cq, tag)); - top_elem->filter->start_transport_op(top_elem, op); -} diff --git a/src/core/lib/surface/lame_client.cc b/src/core/lib/surface/lame_client.cc index 723ec6b703c..8e6f44cd646 100644 --- a/src/core/lib/surface/lame_client.cc +++ b/src/core/lib/surface/lame_client.cc @@ -23,6 +23,7 @@ #include #include +#include "absl/status/status.h" #include "absl/status/statusor.h" #include @@ -53,8 +54,6 @@ // Avoid some IWYU confusion: // IWYU pragma: no_include "src/core/lib/gprpp/orphanable.h" -#define GRPC_ARG_LAME_FILTER_ERROR "grpc.lame_filter_error" - namespace grpc_core { const grpc_channel_filter LameClientFilter::kFilter = @@ -126,11 +125,11 @@ void* ErrorCopy(void* p) { void ErrorDestroy(void* p) { delete static_cast(p); } int ErrorCompare(void* p, void* q) { return QsortCompare(p, q); } +} // namespace + const grpc_arg_pointer_vtable kLameFilterErrorArgVtable = { ErrorCopy, ErrorDestroy, ErrorCompare}; -} // namespace - grpc_arg MakeLameClientErrorArg(grpc_error_handle* error) { return grpc_channel_arg_pointer_create( const_cast(GRPC_ARG_LAME_FILTER_ERROR), error, @@ -138,27 +137,3 @@ grpc_arg MakeLameClientErrorArg(grpc_error_handle* error) { } } // namespace grpc_core - -grpc_channel* grpc_lame_client_channel_create(const char* target, - grpc_status_code error_code, - const char* error_message) { - grpc_core::ExecCtx exec_ctx; - GRPC_API_TRACE( - "grpc_lame_client_channel_create(target=%s, error_code=%d, " - "error_message=%s)", - 3, (target, (int)error_code, error_message)); - if (error_code == GRPC_STATUS_OK) error_code = GRPC_STATUS_UNKNOWN; - grpc_core::ChannelArgs args = - grpc_core::CoreConfiguration::Get() - .channel_args_preconditioning() - .PreconditionChannelArgs(nullptr) - .Set(GRPC_ARG_LAME_FILTER_ERROR, - grpc_core::ChannelArgs::Pointer( - new absl::Status(static_cast(error_code), - error_message), - &grpc_core::kLameFilterErrorArgVtable)); - auto channel = grpc_core::Channel::Create(target, std::move(args), - GRPC_CLIENT_LAME_CHANNEL, nullptr); - GPR_ASSERT(channel.ok()); - return channel->release()->c_ptr(); -} diff --git a/src/core/lib/surface/lame_client.h b/src/core/lib/surface/lame_client.h index 419df7c492f..1a6b423ff5f 100644 --- a/src/core/lib/surface/lame_client.h +++ b/src/core/lib/surface/lame_client.h @@ -38,9 +38,9 @@ #include "src/core/lib/transport/connectivity_state.h" #include "src/core/lib/transport/transport.h" +#define GRPC_ARG_LAME_FILTER_ERROR "grpc.lame_filter_error" + namespace grpc_core { -// Does NOT take ownership of error. -grpc_arg MakeLameClientErrorArg(grpc_error_handle* error); // This filter becomes the entire channel stack for a channel that fails to be // created. Every call returns failure. @@ -66,6 +66,11 @@ class LameClientFilter : public ChannelFilter { }; std::unique_ptr state_; }; + +extern const grpc_arg_pointer_vtable kLameFilterErrorArgVtable; + +grpc_arg MakeLameClientErrorArg(grpc_error_handle* error); + } // namespace grpc_core #endif // GRPC_SRC_CORE_LIB_SURFACE_LAME_CLIENT_H diff --git a/src/core/lib/surface/legacy_channel.cc b/src/core/lib/surface/legacy_channel.cc new file mode 100644 index 00000000000..4a6bc9fcf9c --- /dev/null +++ b/src/core/lib/surface/legacy_channel.cc @@ -0,0 +1,418 @@ +// +// +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// + +#include + +#include "src/core/lib/surface/legacy_channel.h" + +#include "absl/base/thread_annotations.h" +#include "absl/status/status.h" +#include "absl/types/optional.h" + +#include +#include +#include +#include +#include + +#include "src/core/client_channel/client_channel_filter.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/channel/channel_fwd.h" +#include "src/core/lib/channel/channel_stack.h" +#include "src/core/lib/channel/channel_stack_builder_impl.h" +#include "src/core/lib/channel/channelz.h" +#include "src/core/lib/channel/metrics.h" +#include "src/core/lib/config/core_configuration.h" +#include "src/core/lib/debug/stats.h" +#include "src/core/lib/debug/stats_data.h" +#include "src/core/lib/gprpp/crash.h" +#include "src/core/lib/gprpp/dual_ref_counted.h" +#include "src/core/lib/gprpp/ref_counted_ptr.h" +#include "src/core/lib/gprpp/sync.h" +#include "src/core/lib/gprpp/time.h" +#include "src/core/lib/iomgr/closure.h" +#include "src/core/lib/iomgr/error.h" +#include "src/core/lib/iomgr/exec_ctx.h" +#include "src/core/lib/surface/call.h" +#include "src/core/lib/surface/channel.h" +#include "src/core/lib/surface/channel_init.h" +#include "src/core/lib/surface/channel_stack_type.h" +#include "src/core/lib/surface/completion_queue.h" +#include "src/core/lib/surface/init_internally.h" +#include "src/core/lib/surface/lame_client.h" +#include "src/core/lib/transport/call_factory.h" +#include "src/core/lib/transport/transport.h" + +namespace grpc_core { + +absl::StatusOr> LegacyChannel::Create( + std::string target, ChannelArgs args, + grpc_channel_stack_type channel_stack_type) { + if (grpc_channel_stack_type_is_client(channel_stack_type)) { + auto channel_args_mutator = + grpc_channel_args_get_client_channel_creation_mutator(); + if (channel_args_mutator != nullptr) { + args = channel_args_mutator(target.c_str(), args, channel_stack_type); + } + } + ChannelStackBuilderImpl builder( + grpc_channel_stack_type_string(channel_stack_type), channel_stack_type, + args); + builder.SetTarget(target.c_str()); + if (!CoreConfiguration::Get().channel_init().CreateStack(&builder)) { + return nullptr; + } + // Only need to update stats for server channels here. Stats for client + // channels are handled in our base class. + if (builder.channel_stack_type() == GRPC_SERVER_CHANNEL) { + global_stats().IncrementServerChannelsCreated(); + } + absl::StatusOr> r = builder.Build(); + if (!r.ok()) { + auto status = r.status(); + gpr_log(GPR_ERROR, "channel stack builder failed: %s", + status.ToString().c_str()); + return status; + } + // TODO(roth): Figure out how to populate authority here. + // Or maybe just don't worry about this if no one needs it until after + // the call v3 stack lands. + StatsPlugin::ChannelScope scope(builder.target(), ""); + *(*r)->stats_plugin_group = + GlobalStatsPluginRegistry::GetStatsPluginsForChannel(scope); + return MakeOrphanable( + grpc_channel_stack_type_is_client(builder.channel_stack_type()), + builder.IsPromising(), std::move(target), args, std::move(*r)); +} + +namespace { + +class NotReallyACallFactory final : public CallFactory { + public: + using CallFactory::CallFactory; + CallInitiator CreateCall(ClientMetadataHandle, Arena*) override { + Crash("NotReallyACallFactory::CreateCall should never be called"); + } +}; + +} // namespace + +LegacyChannel::LegacyChannel(bool is_client, bool is_promising, + std::string target, + const ChannelArgs& channel_args, + RefCountedPtr channel_stack) + : Channel(std::move(target), channel_args), + is_client_(is_client), + is_promising_(is_promising), + channel_stack_(std::move(channel_stack)), + call_factory_(MakeRefCounted(channel_args)) { + // We need to make sure that grpc_shutdown() does not shut things down + // until after the channel is destroyed. However, the channel may not + // actually be destroyed by the time grpc_channel_destroy() returns, + // since there may be other existing refs to the channel. If those + // refs are held by things that are visible to the wrapped language + // (such as outstanding calls on the channel), then the wrapped + // language can be responsible for making sure that grpc_shutdown() + // does not run until after those refs are released. However, the + // channel may also have refs to itself held internally for various + // things that need to be cleaned up at channel destruction (e.g., + // LB policies, subchannels, etc), and because these refs are not + // visible to the wrapped language, it cannot be responsible for + // deferring grpc_shutdown() until after they are released. To + // accommodate that, we call grpc_init() here and then call + // grpc_shutdown() when the channel is actually destroyed, thus + // ensuring that shutdown is deferred until that point. + InitInternally(); + RefCountedPtr node; + if (channelz_node() != nullptr) { + node = channelz_node()->RefAsSubclass(); + } + *channel_stack_->on_destroy = [node = std::move(node)]() { + if (node != nullptr) { + node->AddTraceEvent(channelz::ChannelTrace::Severity::Info, + grpc_slice_from_static_string("Channel destroyed")); + } + ShutdownInternally(); + }; +} + +void LegacyChannel::Orphan() { + grpc_transport_op* op = grpc_make_transport_op(nullptr); + op->disconnect_with_error = GRPC_ERROR_CREATE("Channel Destroyed"); + grpc_channel_element* elem = + grpc_channel_stack_element(channel_stack_.get(), 0); + elem->filter->start_transport_op(elem, op); + Unref(); +} + +bool LegacyChannel::IsLame() const { + grpc_channel_element* elem = + grpc_channel_stack_last_element(channel_stack_.get()); + return elem->filter == &LameClientFilter::kFilter; +} + +grpc_call* LegacyChannel::CreateCall( + grpc_call* parent_call, uint32_t propagation_mask, + grpc_completion_queue* cq, grpc_pollset_set* pollset_set_alternative, + Slice path, absl::optional authority, Timestamp deadline, + bool registered_method) { + GPR_ASSERT(is_client_); + GPR_ASSERT(!(cq != nullptr && pollset_set_alternative != nullptr)); + grpc_call_create_args args; + args.channel = Ref(); + args.server = nullptr; + args.parent = parent_call; + args.propagation_mask = propagation_mask; + args.cq = cq; + args.pollset_set_alternative = pollset_set_alternative; + args.server_transport_data = nullptr; + args.path = std::move(path); + args.authority = std::move(authority); + args.send_deadline = deadline; + args.registered_method = registered_method; + grpc_call* call; + GRPC_LOG_IF_ERROR("call_create", grpc_call_create(&args, &call)); + return call; +} + +grpc_connectivity_state LegacyChannel::CheckConnectivityState( + bool try_to_connect) { + // Forward through to the underlying client channel. + ClientChannelFilter* client_channel = GetClientChannelFilter(); + if (GPR_UNLIKELY(client_channel == nullptr)) { + if (IsLame()) return GRPC_CHANNEL_TRANSIENT_FAILURE; + gpr_log(GPR_ERROR, + "grpc_channel_check_connectivity_state called on something that is " + "not a client channel"); + return GRPC_CHANNEL_SHUTDOWN; + } + return client_channel->CheckConnectivityState(try_to_connect); +} + +bool LegacyChannel::SupportsConnectivityWatcher() const { + return GetClientChannelFilter() != nullptr; +} + +// A fire-and-forget object to handle external connectivity state watches. +class LegacyChannel::StateWatcher : public DualRefCounted { + public: + StateWatcher(RefCountedPtr channel, grpc_completion_queue* cq, + void* tag, grpc_connectivity_state last_observed_state, + Timestamp deadline) + : channel_(std::move(channel)), + cq_(cq), + tag_(tag), + state_(last_observed_state) { + GPR_ASSERT(grpc_cq_begin_op(cq, tag)); + GRPC_CLOSURE_INIT(&on_complete_, WatchComplete, this, nullptr); + ClientChannelFilter* client_channel = channel_->GetClientChannelFilter(); + if (client_channel == nullptr) { + // If the target URI used to create the channel was invalid, channel + // stack initialization failed, and that caused us to create a lame + // channel. In that case, connectivity state will never change (it + // will always be TRANSIENT_FAILURE), so we don't actually start a + // watch, but we are hiding that fact from the application. + if (channel_->IsLame()) { + // A ref is held by the timer callback. + StartTimer(deadline); + // Ref from object creation needs to be freed here since lame channel + // does not have a watcher. + Unref(); + return; + } + Crash( + "grpc_channel_watch_connectivity_state called on something that is " + "not a client channel"); + } + // Ref from object creation is held by the watcher callback. + auto* watcher_timer_init_state = new WatcherTimerInitState(this, deadline); + client_channel->AddExternalConnectivityWatcher( + grpc_polling_entity_create_from_pollset(grpc_cq_pollset(cq)), &state_, + &on_complete_, watcher_timer_init_state->closure()); + } + + private: + // A fire-and-forget object used to delay starting the timer until the + // ClientChannelFilter actually starts the watch. + class WatcherTimerInitState { + public: + WatcherTimerInitState(StateWatcher* state_watcher, Timestamp deadline) + : state_watcher_(state_watcher), deadline_(deadline) { + GRPC_CLOSURE_INIT(&closure_, WatcherTimerInit, this, nullptr); + } + + grpc_closure* closure() { return &closure_; } + + private: + static void WatcherTimerInit(void* arg, grpc_error_handle /*error*/) { + auto* self = static_cast(arg); + self->state_watcher_->StartTimer(self->deadline_); + delete self; + } + + StateWatcher* state_watcher_; + Timestamp deadline_; + grpc_closure closure_; + }; + + void StartTimer(Timestamp deadline) { + const Duration timeout = deadline - Timestamp::Now(); + MutexLock lock(&mu_); + timer_handle_ = + channel_->event_engine()->RunAfter(timeout, [self = Ref()]() mutable { + ApplicationCallbackExecCtx callback_exec_ctx; + ExecCtx exec_ctx; + self->TimeoutComplete(); + // StateWatcher deletion might require an active ExecCtx. + self.reset(); + }); + } + + void TimeoutComplete() { + timer_fired_ = true; + // If this is a client channel (not a lame channel), cancel the watch. + ClientChannelFilter* client_channel = channel_->GetClientChannelFilter(); + if (client_channel != nullptr) { + client_channel->CancelExternalConnectivityWatcher(&on_complete_); + } + } + + static void WatchComplete(void* arg, grpc_error_handle error) { + RefCountedPtr self(static_cast(arg)); + if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures)) { + GRPC_LOG_IF_ERROR("watch_completion_error", error); + } + MutexLock lock(&self->mu_); + if (self->timer_handle_.has_value()) { + self->channel_->event_engine()->Cancel(*self->timer_handle_); + } + } + + // Invoked when both strong refs are released. + void Orphan() override { + WeakRef().release(); // Take a weak ref until completion is finished. + grpc_error_handle error = + timer_fired_ + ? GRPC_ERROR_CREATE("Timed out waiting for connection state change") + : absl::OkStatus(); + grpc_cq_end_op(cq_, tag_, error, FinishedCompletion, this, + &completion_storage_); + } + + // Called when the completion is returned to the CQ. + static void FinishedCompletion(void* arg, grpc_cq_completion* /*ignored*/) { + auto* self = static_cast(arg); + self->WeakUnref(); + } + + RefCountedPtr channel_; + grpc_completion_queue* cq_; + void* tag_; + + grpc_connectivity_state state_; + grpc_cq_completion completion_storage_; + grpc_closure on_complete_; + + // timer_handle_ might be accessed in parallel from multiple threads, e.g. + // timer callback fired immediately on an EventEngine thread before + // RunAfter() returns. + Mutex mu_; + absl::optional + timer_handle_ ABSL_GUARDED_BY(mu_); + bool timer_fired_ = false; +}; + +void LegacyChannel::WatchConnectivityState( + grpc_connectivity_state last_observed_state, Timestamp deadline, + grpc_completion_queue* cq, void* tag) { + new StateWatcher(RefAsSubclass(), cq, tag, last_observed_state, + deadline); +} + +void LegacyChannel::AddConnectivityWatcher( + grpc_connectivity_state initial_state, + OrphanablePtr watcher) { + auto* client_channel = GetClientChannelFilter(); + GPR_ASSERT(client_channel != nullptr); + client_channel->AddConnectivityWatcher(initial_state, std::move(watcher)); +} + +void LegacyChannel::RemoveConnectivityWatcher( + AsyncConnectivityStateWatcherInterface* watcher) { + auto* client_channel = GetClientChannelFilter(); + GPR_ASSERT(client_channel != nullptr); + client_channel->RemoveConnectivityWatcher(watcher); +} + +void LegacyChannel::GetInfo(const grpc_channel_info* channel_info) { + grpc_channel_element* elem = + grpc_channel_stack_element(channel_stack_.get(), 0); + elem->filter->get_channel_info(elem, channel_info); +} + +void LegacyChannel::ResetConnectionBackoff() { + grpc_transport_op* op = grpc_make_transport_op(nullptr); + op->reset_connect_backoff = true; + grpc_channel_element* elem = + grpc_channel_stack_element(channel_stack_.get(), 0); + elem->filter->start_transport_op(elem, op); +} + +namespace { + +struct ping_result { + grpc_closure closure; + void* tag; + grpc_completion_queue* cq; + grpc_cq_completion completion_storage; +}; +void ping_destroy(void* arg, grpc_cq_completion* /*storage*/) { gpr_free(arg); } + +void ping_done(void* arg, grpc_error_handle error) { + ping_result* pr = static_cast(arg); + grpc_cq_end_op(pr->cq, pr->tag, error, ping_destroy, pr, + &pr->completion_storage); +} + +} // namespace + +void LegacyChannel::Ping(grpc_completion_queue* cq, void* tag) { + ping_result* pr = static_cast(gpr_malloc(sizeof(*pr))); + pr->tag = tag; + pr->cq = cq; + GRPC_CLOSURE_INIT(&pr->closure, ping_done, pr, grpc_schedule_on_exec_ctx); + grpc_transport_op* op = grpc_make_transport_op(nullptr); + op->send_ping.on_ack = &pr->closure; + op->bind_pollset = grpc_cq_pollset(cq); + GPR_ASSERT(grpc_cq_begin_op(cq, tag)); + grpc_channel_element* top_elem = + grpc_channel_stack_element(channel_stack_.get(), 0); + top_elem->filter->start_transport_op(top_elem, op); +} + +ClientChannelFilter* LegacyChannel::GetClientChannelFilter() const { + grpc_channel_element* elem = + grpc_channel_stack_last_element(channel_stack_.get()); + if (elem->filter != &ClientChannelFilter::kFilterVtableWithPromises && + elem->filter != &ClientChannelFilter::kFilterVtableWithoutPromises) { + return nullptr; + } + return static_cast(elem->channel_data); +} + +} // namespace grpc_core diff --git a/src/core/lib/surface/legacy_channel.h b/src/core/lib/surface/legacy_channel.h new file mode 100644 index 00000000000..cda0dbc6060 --- /dev/null +++ b/src/core/lib/surface/legacy_channel.h @@ -0,0 +1,118 @@ +// +// +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// + +#ifndef GRPC_SRC_CORE_LIB_SURFACE_LEGACY_CHANNEL_H +#define GRPC_SRC_CORE_LIB_SURFACE_LEGACY_CHANNEL_H + +#include + +#include + +#include "absl/status/statusor.h" +#include "absl/types/optional.h" + +#include +#include + +#include "src/core/client_channel/client_channel_filter.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/channel/channel_fwd.h" +#include "src/core/lib/channel/channel_stack.h" // IWYU pragma: keep +#include "src/core/lib/gprpp/ref_counted_ptr.h" +#include "src/core/lib/gprpp/time.h" +#include "src/core/lib/iomgr/iomgr_fwd.h" +#include "src/core/lib/slice/slice.h" +#include "src/core/lib/surface/channel.h" +#include "src/core/lib/surface/channel_stack_type.h" +#include "src/core/lib/transport/call_factory.h" +#include "src/core/lib/transport/transport.h" + +namespace grpc_core { + +class LegacyChannel : public Channel { + public: + static absl::StatusOr> Create( + std::string target, ChannelArgs args, + grpc_channel_stack_type channel_stack_type); + + // Do not instantiate directly -- use Create() instead. + LegacyChannel(bool is_client, bool is_promising, std::string target, + const ChannelArgs& channel_args, + RefCountedPtr channel_stack); + + void Orphan() override; + + Arena* CreateArena() override { return call_factory_->CreateArena(); } + void DestroyArena(Arena* arena) override { + return call_factory_->DestroyArena(arena); + } + + bool IsLame() const override; + + grpc_call* CreateCall(grpc_call* parent_call, uint32_t propagation_mask, + grpc_completion_queue* cq, + grpc_pollset_set* pollset_set_alternative, Slice path, + absl::optional authority, Timestamp deadline, + bool registered_method) override; + + grpc_event_engine::experimental::EventEngine* event_engine() const override { + return channel_stack_->EventEngine(); + } + + bool SupportsConnectivityWatcher() const override; + + grpc_connectivity_state CheckConnectivityState(bool try_to_connect) override; + + void WatchConnectivityState(grpc_connectivity_state last_observed_state, + Timestamp deadline, grpc_completion_queue* cq, + void* tag) override; + + void AddConnectivityWatcher( + grpc_connectivity_state initial_state, + OrphanablePtr watcher) override; + void RemoveConnectivityWatcher( + AsyncConnectivityStateWatcherInterface* watcher) override; + + void GetInfo(const grpc_channel_info* channel_info) override; + + void ResetConnectionBackoff() override; + + void Ping(grpc_completion_queue* cq, void* tag) override; + + bool is_client() const override { return is_client_; } + bool is_promising() const override { return is_promising_; } + grpc_channel_stack* channel_stack() const override { + return channel_stack_.get(); + } + + private: + class StateWatcher; + + // Returns the client channel filter if this is a client channel, + // otherwise null. + ClientChannelFilter* GetClientChannelFilter() const; + + const bool is_client_; + const bool is_promising_; + RefCountedPtr channel_stack_; + const RefCountedPtr call_factory_; +}; + +} // namespace grpc_core + +#endif // GRPC_SRC_CORE_LIB_SURFACE_LEGACY_CHANNEL_H diff --git a/src/core/lib/surface/server.cc b/src/core/lib/surface/server.cc index 06c9ec9ee2d..7cdd88dcdae 100644 --- a/src/core/lib/surface/server.cc +++ b/src/core/lib/surface/server.cc @@ -77,6 +77,7 @@ #include "src/core/lib/surface/channel.h" #include "src/core/lib/surface/channel_stack_type.h" #include "src/core/lib/surface/completion_queue.h" +#include "src/core/lib/surface/legacy_channel.h" #include "src/core/lib/surface/wait_for_cq_end_op.h" #include "src/core/lib/transport/connectivity_state.h" #include "src/core/lib/transport/error_utils.h" @@ -742,7 +743,7 @@ class ChannelBroadcaster { // Broadcasts a shutdown on each channel. void BroadcastShutdown(bool send_goaway, grpc_error_handle force_disconnect) { for (const RefCountedPtr& channel : channels_) { - SendShutdown(channel->c_ptr(), send_goaway, force_disconnect); + SendShutdown(channel.get(), send_goaway, force_disconnect); } channels_.clear(); // just for safety against double broadcast } @@ -759,7 +760,7 @@ class ChannelBroadcaster { delete a; } - static void SendShutdown(grpc_channel* channel, bool send_goaway, + static void SendShutdown(Channel* channel, bool send_goaway, grpc_error_handle send_disconnect) { ShutdownCleanupArgs* sc = new ShutdownCleanupArgs; GRPC_CLOSURE_INIT(&sc->closure, ShutdownCleanup, sc, @@ -773,8 +774,7 @@ class ChannelBroadcaster { : absl::OkStatus(); sc->slice = grpc_slice_from_copied_string("Server shutdown"); op->disconnect_with_error = send_disconnect; - elem = - grpc_channel_stack_element(grpc_channel_get_channel_stack(channel), 0); + elem = grpc_channel_stack_element(channel->channel_stack(), 0); elem->filter->start_transport_op(elem, op); } @@ -911,8 +911,9 @@ grpc_error_handle Server::SetupTransport( const ChannelArgs& args, const RefCountedPtr& socket_node) { // Create channel. - absl::StatusOr> channel = - Channel::Create(nullptr, args, GRPC_SERVER_CHANNEL, transport); + global_stats().IncrementServerChannelsCreated(); + absl::StatusOr> channel = + LegacyChannel::Create("", args.SetObject(transport), GRPC_SERVER_CHANNEL); if (!channel.ok()) { return absl_status_to_grpc_error(channel.status()); } @@ -1309,11 +1310,11 @@ absl::StatusOr Server::ChannelData::CreateCall( } void Server::ChannelData::InitTransport(RefCountedPtr server, - RefCountedPtr channel, + OrphanablePtr channel, size_t cq_idx, Transport* transport, intptr_t channelz_socket_uuid) { server_ = std::move(server); - channel_ = channel; + channel_ = std::move(channel); cq_idx_ = cq_idx; channelz_socket_uuid_ = channelz_socket_uuid; // Publish channel. @@ -1392,7 +1393,7 @@ void Server::ChannelData::AcceptStream(void* arg, Transport* /*transport*/, auto* chand = static_cast(arg); // create a call grpc_call_create_args args; - args.channel = chand->channel_; + args.channel = chand->channel_->Ref(); args.server = chand->server_.get(); args.parent = nullptr; args.propagation_mask = 0; diff --git a/src/core/lib/surface/server.h b/src/core/lib/surface/server.h index 4bb6fce3fae..3623ffa0653 100644 --- a/src/core/lib/surface/server.h +++ b/src/core/lib/surface/server.h @@ -66,6 +66,7 @@ #include "src/core/lib/slice/slice.h" #include "src/core/lib/surface/channel.h" #include "src/core/lib/surface/completion_queue.h" +#include "src/core/lib/surface/server_interface.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" @@ -77,7 +78,8 @@ namespace grpc_core { extern TraceFlag grpc_server_channel_trace; -class Server : public InternallyRefCounted, +class Server : public ServerInterface, + public InternallyRefCounted, public CppImplOf { public: // Filter vtable. @@ -134,8 +136,10 @@ class Server : public InternallyRefCounted, void Orphan() ABSL_LOCKS_EXCLUDED(mu_global_) override; - const ChannelArgs& channel_args() const { return channel_args_; } - channelz::ServerNode* channelz_node() const { return channelz_node_.get(); } + const ChannelArgs& channel_args() const override { return channel_args_; } + channelz::ServerNode* channelz_node() const override { + return channelz_node_.get(); + } // Do not call this before Start(). Returns the pollsets. The // vector itself is immutable, but the pollsets inside are mutable. The @@ -146,7 +150,7 @@ class Server : public InternallyRefCounted, return config_fetcher_.get(); } - ServerCallTracerFactory* server_call_tracer_factory() const { + ServerCallTracerFactory* server_call_tracer_factory() const override { return server_call_tracer_factory_; } @@ -224,7 +228,7 @@ class Server : public InternallyRefCounted, ~ChannelData(); void InitTransport(RefCountedPtr server, - RefCountedPtr channel, size_t cq_idx, + OrphanablePtr channel, size_t cq_idx, Transport* transport, intptr_t channelz_socket_uuid); RefCountedPtr server() const { return server_; } @@ -257,7 +261,7 @@ class Server : public InternallyRefCounted, static void FinishDestroy(void* arg, grpc_error_handle error); RefCountedPtr server_; - RefCountedPtr channel_; + OrphanablePtr channel_; // The index into Server::cqs_ of the CQ used as a starting point for // where to publish new incoming calls. size_t cq_idx_; diff --git a/src/core/lib/surface/server_interface.h b/src/core/lib/surface/server_interface.h new file mode 100644 index 00000000000..ea1081287d2 --- /dev/null +++ b/src/core/lib/surface/server_interface.h @@ -0,0 +1,43 @@ +// +// Copyright 2024 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#ifndef GRPC_SRC_CORE_LIB_SURFACE_SERVER_INTERFACE_H +#define GRPC_SRC_CORE_LIB_SURFACE_SERVER_INTERFACE_H + +#include + +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/channel/channelz.h" + +namespace grpc_core { + +class ServerCallTracerFactory; + +// This class is a hack to avoid a circular dependency that would be +// caused by the code in call.cc depending directly on the server code. +// TODO(roth): After the call v3 migration, find a cleaner way to do this. +class ServerInterface { + public: + virtual ~ServerInterface() = default; + + virtual const ChannelArgs& channel_args() const = 0; + virtual channelz::ServerNode* channelz_node() const = 0; + virtual ServerCallTracerFactory* server_call_tracer_factory() const = 0; +}; + +} // namespace grpc_core + +#endif // GRPC_SRC_CORE_LIB_SURFACE_SERVER_INTERFACE_H diff --git a/src/core/load_balancing/grpclb/grpclb.cc b/src/core/load_balancing/grpclb/grpclb.cc index f739ae74ee2..8f321f496fe 100644 --- a/src/core/load_balancing/grpclb/grpclb.cc +++ b/src/core/load_balancing/grpclb/grpclb.cc @@ -547,7 +547,7 @@ class GrpcLb : public LoadBalancingPolicy { bool shutting_down_ = false; // The channel for communicating with the LB server. - grpc_channel* lb_channel_ = nullptr; + OrphanablePtr lb_channel_; StateWatcher* watcher_ = nullptr; // Response generator to inject address updates into lb_channel_. RefCountedPtr response_generator_; @@ -898,11 +898,11 @@ GrpcLb::BalancerCallState::BalancerCallState( grpclb_policy()->lb_call_timeout_ == Duration::Zero() ? Timestamp::InfFuture() : Timestamp::Now() + grpclb_policy()->lb_call_timeout_; - lb_call_ = grpc_channel_create_pollset_set_call( - grpclb_policy()->lb_channel_, nullptr, GRPC_PROPAGATE_DEFAULTS, - grpclb_policy_->interested_parties(), - Slice::FromStaticString("/grpc.lb.v1.LoadBalancer/BalanceLoad").c_slice(), - nullptr, deadline, nullptr); + lb_call_ = grpclb_policy()->lb_channel_->CreateCall( + /*parent_call=*/nullptr, GRPC_PROPAGATE_DEFAULTS, + /*cq=*/nullptr, grpclb_policy_->interested_parties(), + Slice::FromStaticString("/grpc.lb.v1.LoadBalancer/BalanceLoad"), + /*authority=*/absl::nullopt, deadline, /*registered_method=*/true); // Init the LB call request payload. upb::Arena arena; grpc_slice request_payload_slice = GrpcLbRequestCreate( @@ -1504,13 +1504,11 @@ void GrpcLb::ShutdownLocked() { // alive when that callback is invoked. if (lb_channel_ != nullptr) { if (parent_channelz_node_ != nullptr) { - channelz::ChannelNode* child_channelz_node = - grpc_channel_get_channelz_node(lb_channel_); + channelz::ChannelNode* child_channelz_node = lb_channel_->channelz_node(); GPR_ASSERT(child_channelz_node != nullptr); parent_channelz_node_->RemoveChildChannel(child_channelz_node->uuid()); } - grpc_channel_destroy_internal(lb_channel_); - lb_channel_ = nullptr; + lb_channel_.reset(); } } @@ -1520,7 +1518,7 @@ void GrpcLb::ShutdownLocked() { void GrpcLb::ResetBackoffLocked() { if (lb_channel_ != nullptr) { - grpc_channel_reset_connect_backoff(lb_channel_); + lb_channel_->ResetConnectionBackoff(); } if (child_policy_ != nullptr) { child_policy_->ResetBackoffLocked(); @@ -1592,13 +1590,9 @@ absl::Status GrpcLb::UpdateLocked(UpdateArgs args) { // Start watching the channel's connectivity state. If the channel // goes into state TRANSIENT_FAILURE before the timer fires, we go into // fallback mode even if the fallback timeout has not elapsed. - ClientChannelFilter* client_channel = - ClientChannelFilter::GetFromChannel(Channel::FromC(lb_channel_)); - GPR_ASSERT(client_channel != nullptr); - // Ref held by callback. watcher_ = new StateWatcher(RefAsSubclass(DEBUG_LOCATION, "StateWatcher")); - client_channel->AddConnectivityWatcher( + lb_channel_->AddConnectivityWatcher( GRPC_CHANNEL_IDLE, OrphanablePtr(watcher_)); // Start balancer call. @@ -1633,13 +1627,13 @@ absl::Status GrpcLb::UpdateBalancerChannelLocked() { if (lb_channel_ == nullptr) { std::string uri_str = absl::StrCat("fake:///", channel_control_helper()->GetAuthority()); - lb_channel_ = - grpc_channel_create(uri_str.c_str(), channel_credentials.get(), - lb_channel_args.ToC().get()); + lb_channel_.reset( + Channel::FromC( + grpc_channel_create(uri_str.c_str(), channel_credentials.get(), + lb_channel_args.ToC().get()))); GPR_ASSERT(lb_channel_ != nullptr); // Set up channelz linkage. - channelz::ChannelNode* child_channelz_node = - grpc_channel_get_channelz_node(lb_channel_); + channelz::ChannelNode* child_channelz_node = lb_channel_->channelz_node(); auto parent_channelz_node = args_.GetObjectRef(); if (child_channelz_node != nullptr && parent_channelz_node != nullptr) { parent_channelz_node->AddChildChannel(child_channelz_node->uuid()); @@ -1659,10 +1653,7 @@ absl::Status GrpcLb::UpdateBalancerChannelLocked() { } void GrpcLb::CancelBalancerChannelConnectivityWatchLocked() { - ClientChannelFilter* client_channel = - ClientChannelFilter::GetFromChannel(Channel::FromC(lb_channel_)); - GPR_ASSERT(client_channel != nullptr); - client_channel->RemoveConnectivityWatcher(watcher_); + lb_channel_->RemoveConnectivityWatcher(watcher_); } // @@ -1678,7 +1669,7 @@ void GrpcLb::StartBalancerCallLocked() { if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_glb_trace)) { gpr_log(GPR_INFO, "[grpclb %p] Query for backends (lb_channel: %p, lb_calld: %p)", - this, lb_channel_, lb_calld_.get()); + this, lb_channel_.get(), lb_calld_.get()); } lb_calld_->StartQuery(); } diff --git a/src/core/load_balancing/rls/rls.cc b/src/core/load_balancing/rls/rls.cc index e62efb4a1f3..4a90a17cfe7 100644 --- a/src/core/load_balancing/rls/rls.cc +++ b/src/core/load_balancing/rls/rls.cc @@ -568,7 +568,7 @@ class RlsLb : public LoadBalancingPolicy { // Resets the channel's backoff. void ResetBackoff(); - grpc_channel* channel() const { return channel_; } + Channel* channel() const { return channel_.get(); } private: // Watches the state of the RLS channel. Notifies the LB policy when @@ -620,7 +620,7 @@ class RlsLb : public LoadBalancingPolicy { RefCountedPtr lb_policy_; bool is_shutdown_ = false; - grpc_channel* channel_ = nullptr; + OrphanablePtr channel_; RefCountedPtr parent_channelz_node_; StateWatcher* watcher_ = nullptr; Throttle throttle_ ABSL_GUARDED_BY(&RlsLb::mu_); @@ -1539,17 +1539,18 @@ RlsLb::RlsChannel::RlsChannel(RefCountedPtr lb_policy) args = args.Set(GRPC_ARG_SERVICE_CONFIG, service_config) .Set(GRPC_ARG_SERVICE_CONFIG_DISABLE_RESOLUTION, 1); } - channel_ = grpc_channel_create(lb_policy_->config_->lookup_service().c_str(), - creds.get(), args.ToC().get()); + channel_.reset( + Channel::FromC( + grpc_channel_create(lb_policy_->config_->lookup_service().c_str(), + creds.get(), args.ToC().get()))); if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) { gpr_log(GPR_INFO, "[rlslb %p] RlsChannel=%p: created channel %p for %s", - lb_policy_.get(), this, channel_, + lb_policy_.get(), this, channel_.get(), lb_policy_->config_->lookup_service().c_str()); } if (channel_ != nullptr) { // Set up channelz linkage. - channelz::ChannelNode* child_channelz_node = - grpc_channel_get_channelz_node(channel_); + channelz::ChannelNode* child_channelz_node = channel_->channelz_node(); auto parent_channelz_node = lb_policy_->channel_args_.GetObjectRef(); if (child_channelz_node != nullptr && parent_channelz_node != nullptr) { @@ -1557,11 +1558,8 @@ RlsLb::RlsChannel::RlsChannel(RefCountedPtr lb_policy) parent_channelz_node_ = std::move(parent_channelz_node); } // Start connectivity watch. - ClientChannelFilter* client_channel = - ClientChannelFilter::GetFromChannel(Channel::FromC(channel_)); - GPR_ASSERT(client_channel != nullptr); watcher_ = new StateWatcher(Ref(DEBUG_LOCATION, "StateWatcher")); - client_channel->AddConnectivityWatcher( + channel_->AddConnectivityWatcher( GRPC_CHANNEL_IDLE, OrphanablePtr(watcher_)); } @@ -1570,26 +1568,22 @@ RlsLb::RlsChannel::RlsChannel(RefCountedPtr lb_policy) void RlsLb::RlsChannel::Orphan() { if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) { gpr_log(GPR_INFO, "[rlslb %p] RlsChannel=%p, channel=%p: shutdown", - lb_policy_.get(), this, channel_); + lb_policy_.get(), this, channel_.get()); } is_shutdown_ = true; if (channel_ != nullptr) { // Remove channelz linkage. if (parent_channelz_node_ != nullptr) { - channelz::ChannelNode* child_channelz_node = - grpc_channel_get_channelz_node(channel_); + channelz::ChannelNode* child_channelz_node = channel_->channelz_node(); GPR_ASSERT(child_channelz_node != nullptr); parent_channelz_node_->RemoveChildChannel(child_channelz_node->uuid()); } // Stop connectivity watch. if (watcher_ != nullptr) { - ClientChannelFilter* client_channel = - ClientChannelFilter::GetFromChannel(Channel::FromC(channel_)); - GPR_ASSERT(client_channel != nullptr); - client_channel->RemoveConnectivityWatcher(watcher_); + channel_->RemoveConnectivityWatcher(watcher_); watcher_ = nullptr; } - grpc_channel_destroy_internal(channel_); + channel_.reset(); } Unref(DEBUG_LOCATION, "Orphan"); } @@ -1618,7 +1612,7 @@ void RlsLb::RlsChannel::ReportResponseLocked(bool response_succeeded) { void RlsLb::RlsChannel::ResetBackoff() { GPR_DEBUG_ASSERT(channel_ != nullptr); - grpc_channel_reset_connect_backoff(channel_); + channel_->ResetConnectionBackoff(); } // @@ -1683,11 +1677,11 @@ void RlsLb::RlsRequest::StartCallLocked() { deadline_ = now + lb_policy_->config_->lookup_service_timeout(); grpc_metadata_array_init(&recv_initial_metadata_); grpc_metadata_array_init(&recv_trailing_metadata_); - call_ = grpc_channel_create_pollset_set_call( - rls_channel_->channel(), nullptr, GRPC_PROPAGATE_DEFAULTS, + call_ = rls_channel_->channel()->CreateCall( + /*parent_call=*/nullptr, GRPC_PROPAGATE_DEFAULTS, /*cq=*/nullptr, lb_policy_->interested_parties(), - grpc_slice_from_static_string(kRlsRequestPath), nullptr, deadline_, - nullptr); + Slice::FromStaticString(kRlsRequestPath), /*authority=*/absl::nullopt, + deadline_, /*registered_method=*/true); grpc_op ops[6]; memset(ops, 0, sizeof(ops)); grpc_op* op = ops; diff --git a/src/core/plugin_registry/grpc_plugin_registry.cc b/src/core/plugin_registry/grpc_plugin_registry.cc index 0401a799ba5..9f1c6cf9482 100644 --- a/src/core/plugin_registry/grpc_plugin_registry.cc +++ b/src/core/plugin_registry/grpc_plugin_registry.cc @@ -21,10 +21,13 @@ #include #include "src/core/lib/config/core_configuration.h" -#include "src/core/lib/surface/builtins.h" +#include "src/core/lib/surface/channel_stack_type.h" +#include "src/core/lib/surface/lame_client.h" +#include "src/core/lib/surface/server.h" #include "src/core/lib/transport/http_connect_handshaker.h" #include "src/core/lib/transport/tcp_connect_handshaker.h" + namespace grpc_event_engine { namespace experimental { extern void RegisterEventEngineChannelArgPreconditioning( @@ -73,6 +76,20 @@ extern void RegisterRlsLbPolicy(CoreConfiguration::Builder* builder); extern void RegisterBinderResolver(CoreConfiguration::Builder* builder); #endif +namespace { + +void RegisterBuiltins(CoreConfiguration::Builder* builder) { + RegisterServerCallTracerFilter(builder); + builder->channel_init() + ->RegisterFilter(GRPC_CLIENT_LAME_CHANNEL) + .Terminal(); + builder->channel_init() + ->RegisterFilter(GRPC_SERVER_CHANNEL, &Server::kServerTopFilter) + .BeforeAll(); +} + +} // namespace + void BuildCoreConfiguration(CoreConfiguration::Builder* builder) { grpc_event_engine::experimental::RegisterEventEngineChannelArgPreconditioning( builder); diff --git a/src/core/plugin_registry/grpc_plugin_registry_extra.cc b/src/core/plugin_registry/grpc_plugin_registry_extra.cc index bf7fa8abb50..25e26342c2a 100644 --- a/src/core/plugin_registry/grpc_plugin_registry_extra.cc +++ b/src/core/plugin_registry/grpc_plugin_registry_extra.cc @@ -19,7 +19,6 @@ #include #include "src/core/lib/config/core_configuration.h" -#include "src/core/lib/surface/builtins.h" namespace grpc_core { #ifndef GRPC_NO_XDS diff --git a/src/core/plugin_registry/grpc_plugin_registry_noextra.cc b/src/core/plugin_registry/grpc_plugin_registry_noextra.cc index 5d693f9a9f9..00692f98efd 100644 --- a/src/core/plugin_registry/grpc_plugin_registry_noextra.cc +++ b/src/core/plugin_registry/grpc_plugin_registry_noextra.cc @@ -19,7 +19,6 @@ #include #include "src/core/lib/config/core_configuration.h" -#include "src/core/lib/surface/builtins.h" namespace grpc_core { void RegisterExtraFilters(CoreConfiguration::Builder* /* builder */) {} diff --git a/src/core/tsi/alts/handshaker/alts_handshaker_client.cc b/src/core/tsi/alts/handshaker/alts_handshaker_client.cc index 3dd3d218398..06a3a790cc2 100644 --- a/src/core/tsi/alts/handshaker/alts_handshaker_client.cc +++ b/src/core/tsi/alts/handshaker/alts_handshaker_client.cc @@ -742,10 +742,12 @@ alts_handshaker_client* alts_grpc_handshaker_client_create( strcmp(handshaker_service_url, ALTS_HANDSHAKER_SERVICE_URL_FOR_TESTING) == 0 ? nullptr - : grpc_channel_create_pollset_set_call( - channel, nullptr, GRPC_PROPAGATE_DEFAULTS, interested_parties, - grpc_slice_from_static_string(ALTS_SERVICE_METHOD), nullptr, - grpc_core::Timestamp::InfFuture(), nullptr); + : grpc_core::Channel::FromC(channel)->CreateCall( + /*parent_call=*/nullptr, GRPC_PROPAGATE_DEFAULTS, + /*cq=*/nullptr, interested_parties, + grpc_core::Slice::FromStaticString(ALTS_SERVICE_METHOD), + /*authority=*/absl::nullopt, grpc_core::Timestamp::InfFuture(), + /*registered_method=*/true); GRPC_CLOSURE_INIT(&client->on_handshaker_service_resp_recv, grpc_cb, client, grpc_schedule_on_exec_ctx); GRPC_CLOSURE_INIT(&client->on_status_received, on_status_received, client, diff --git a/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc b/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc index 612fea591da..e34f90b0cb3 100644 --- a/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc +++ b/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc @@ -36,6 +36,7 @@ #include "src/core/lib/gprpp/memory.h" #include "src/core/lib/gprpp/sync.h" #include "src/core/lib/iomgr/closure.h" +#include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/surface/channel.h" #include "src/core/tsi/alts/frame_protector/alts_frame_protector.h" #include "src/core/tsi/alts/handshaker/alts_handshaker_client.h" diff --git a/src/cpp/ext/csm/BUILD b/src/cpp/ext/csm/BUILD index 06e1c266168..e822ea37b65 100644 --- a/src/cpp/ext/csm/BUILD +++ b/src/cpp/ext/csm/BUILD @@ -55,6 +55,7 @@ grpc_cc_library( language = "c++", visibility = ["//:__subpackages__"], deps = [ + "//:call_tracer", "//:gpr", "//:gpr_platform", "//:grpc_base", diff --git a/src/cpp/ext/otel/BUILD b/src/cpp/ext/otel/BUILD index b877af9ff15..d1e8d173d55 100644 --- a/src/cpp/ext/otel/BUILD +++ b/src/cpp/ext/otel/BUILD @@ -59,6 +59,7 @@ grpc_cc_library( language = "c++", visibility = ["//:__subpackages__"], deps = [ + "//:call_tracer", "//:config", "//:gpr", "//:gpr_platform", diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 62a39f405d6..57a24b8d1fb 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -16,7 +16,6 @@ CORE_SOURCE_FILES = [ 'src/core/client_channel/backup_poller.cc', - 'src/core/client_channel/channel_connectivity.cc', 'src/core/client_channel/client_channel_channelz.cc', 'src/core/client_channel/client_channel_factory.cc', 'src/core/client_channel/client_channel_filter.cc', @@ -736,15 +735,14 @@ CORE_SOURCE_FILES = [ 'src/core/lib/slice/slice_refcount.cc', 'src/core/lib/slice/slice_string_helpers.cc', 'src/core/lib/surface/api_trace.cc', - 'src/core/lib/surface/builtins.cc', 'src/core/lib/surface/byte_buffer.cc', 'src/core/lib/surface/byte_buffer_reader.cc', 'src/core/lib/surface/call.cc', 'src/core/lib/surface/call_details.cc', 'src/core/lib/surface/call_log_batch.cc', 'src/core/lib/surface/channel.cc', + 'src/core/lib/surface/channel_create.cc', 'src/core/lib/surface/channel_init.cc', - 'src/core/lib/surface/channel_ping.cc', 'src/core/lib/surface/channel_stack_type.cc', 'src/core/lib/surface/completion_queue.cc', 'src/core/lib/surface/completion_queue_factory.cc', @@ -752,6 +750,7 @@ CORE_SOURCE_FILES = [ 'src/core/lib/surface/init.cc', 'src/core/lib/surface/init_internally.cc', 'src/core/lib/surface/lame_client.cc', + 'src/core/lib/surface/legacy_channel.cc', 'src/core/lib/surface/metadata_array.cc', 'src/core/lib/surface/server.cc', 'src/core/lib/surface/validate_metadata.cc', diff --git a/test/core/bad_connection/BUILD b/test/core/bad_connection/BUILD index 1421b855d9e..6a0ce0c923c 100644 --- a/test/core/bad_connection/BUILD +++ b/test/core/bad_connection/BUILD @@ -28,6 +28,7 @@ grpc_cc_binary( deps = [ "//:gpr", "//:grpc", + "//:server", "//src/core:channel_args", "//test/core/util:grpc_test_util", ], diff --git a/test/core/bad_connection/close_fd_test.cc b/test/core/bad_connection/close_fd_test.cc index 551cb0ddb1e..7313304a05e 100644 --- a/test/core/bad_connection/close_fd_test.cc +++ b/test/core/bad_connection/close_fd_test.cc @@ -39,6 +39,7 @@ #include "src/core/lib/iomgr/error.h" #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/iomgr/port.h" +#include "src/core/lib/surface/channel_create.h" #include "src/core/lib/surface/channel_stack_type.h" #include "src/core/lib/transport/transport.h" @@ -108,9 +109,10 @@ static void client_setup_transport(grpc_core::Transport* transport) { grpc_channel_args_copy_and_add(nullptr, &authority_arg, 1); // TODO (pjaikumar): use GRPC_CLIENT_CHANNEL instead of // GRPC_CLIENT_DIRECT_CHANNEL - g_ctx.client = (*grpc_core::Channel::Create( + g_ctx.client = (*grpc_core::ChannelCreate( "socketpair-target", grpc_core::ChannelArgs::FromC(args), GRPC_CLIENT_DIRECT_CHANNEL, transport)) + .release() ->c_ptr(); grpc_channel_args_destroy(args); } diff --git a/test/core/end2end/fixtures/sockpair_fixture.h b/test/core/end2end/fixtures/sockpair_fixture.h index 236f251b8d0..e8d7c30c4d8 100644 --- a/test/core/end2end/fixtures/sockpair_fixture.h +++ b/test/core/end2end/fixtures/sockpair_fixture.h @@ -38,6 +38,7 @@ #include "src/core/lib/iomgr/error.h" #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/surface/channel.h" +#include "src/core/lib/surface/channel_create.h" #include "src/core/lib/surface/channel_stack_type.h" #include "src/core/lib/surface/completion_queue.h" #include "src/core/lib/surface/server.h" @@ -108,8 +109,8 @@ class SockpairFixture : public CoreTestFixture { auto* client_endpoint = std::exchange(ep_.client, nullptr); EXPECT_NE(client_endpoint, nullptr); transport = grpc_create_chttp2_transport(args, client_endpoint, true); - auto channel = Channel::Create("socketpair-target", args, - GRPC_CLIENT_DIRECT_CHANNEL, transport); + auto channel = ChannelCreate("socketpair-target", args, + GRPC_CLIENT_DIRECT_CHANNEL, transport); grpc_channel* client; if (channel.ok()) { client = channel->release()->c_ptr(); diff --git a/test/core/end2end/fuzzers/client_fuzzer.cc b/test/core/end2end/fuzzers/client_fuzzer.cc index 55400b74766..788d8dc8c2d 100644 --- a/test/core/end2end/fuzzers/client_fuzzer.cc +++ b/test/core/end2end/fuzzers/client_fuzzer.cc @@ -32,6 +32,7 @@ #include "src/core/lib/iomgr/endpoint.h" #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/surface/channel.h" +#include "src/core/lib/surface/channel_create.h" #include "src/core/lib/surface/channel_stack_type.h" #include "src/core/lib/transport/transport.h" #include "src/libfuzzer/libfuzzer_macro.h" @@ -66,8 +67,8 @@ class ClientFuzzer final : public BasicFuzzer { .SetIfUnset(GRPC_ARG_DEFAULT_AUTHORITY, "test-authority"); Transport* transport = grpc_create_chttp2_transport(args, mock_endpoint_, true); - channel_ = Channel::Create("test-target", args, GRPC_CLIENT_DIRECT_CHANNEL, - transport) + channel_ = ChannelCreate("test-target", args, GRPC_CLIENT_DIRECT_CHANNEL, + transport) ->release() ->c_ptr(); } diff --git a/test/core/surface/lame_client_test.cc b/test/core/surface/lame_client_test.cc index 7d514dab2af..8e025c39265 100644 --- a/test/core/surface/lame_client_test.cc +++ b/test/core/surface/lame_client_test.cc @@ -58,8 +58,8 @@ void test_transport_op(grpc_channel* channel) { grpc_core::ExecCtx exec_ctx; grpc_transport_op* op = grpc_make_transport_op(nullptr); op->start_connectivity_watch = grpc_core::MakeOrphanable(); - grpc_channel_element* elem = - grpc_channel_stack_element(grpc_channel_get_channel_stack(channel), 0); + grpc_channel_element* elem = grpc_channel_stack_element( + grpc_core::Channel::FromC(channel)->channel_stack(), 0); elem->filter->start_transport_op(elem, op); GRPC_CLOSURE_INIT(&transport_op_cb, do_nothing, nullptr, diff --git a/test/core/surface/secure_channel_create_test.cc b/test/core/surface/secure_channel_create_test.cc index 1b876bb632b..0b4328f9f93 100644 --- a/test/core/surface/secure_channel_create_test.cc +++ b/test/core/surface/secure_channel_create_test.cc @@ -1,5 +1,4 @@ // -// // Copyright 2015 gRPC authors. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -36,8 +35,8 @@ void test_unknown_scheme_target(void) { grpc_channel_credentials* creds = grpc_fake_transport_security_credentials_create(); grpc_channel* chan = grpc_channel_create("blah://blah", creds, nullptr); - grpc_channel_element* elem = - grpc_channel_stack_element(grpc_channel_get_channel_stack(chan), 0); + grpc_channel_element* elem = grpc_channel_stack_element( + grpc_core::Channel::FromC(chan)->channel_stack(), 0); ASSERT_STREQ(elem->filter->name, "lame-client"); grpc_core::ExecCtx exec_ctx; grpc_core::Channel::FromC(chan)->Unref(); @@ -50,8 +49,8 @@ void test_security_connector_already_in_arg(void) { args.num_args = 1; args.args = &arg; grpc_channel* chan = grpc_channel_create(nullptr, nullptr, &args); - grpc_channel_element* elem = - grpc_channel_stack_element(grpc_channel_get_channel_stack(chan), 0); + grpc_channel_element* elem = grpc_channel_stack_element( + grpc_core::Channel::FromC(chan)->channel_stack(), 0); ASSERT_STREQ(elem->filter->name, "lame-client"); grpc_core::ExecCtx exec_ctx; grpc_core::Channel::FromC(chan)->Unref(); @@ -59,8 +58,8 @@ void test_security_connector_already_in_arg(void) { void test_null_creds(void) { grpc_channel* chan = grpc_channel_create(nullptr, nullptr, nullptr); - grpc_channel_element* elem = - grpc_channel_stack_element(grpc_channel_get_channel_stack(chan), 0); + grpc_channel_element* elem = grpc_channel_stack_element( + grpc_core::Channel::FromC(chan)->channel_stack(), 0); ASSERT_STREQ(elem->filter->name, "lame-client"); grpc_core::ExecCtx exec_ctx; grpc_core::Channel::FromC(chan)->Unref(); diff --git a/test/core/transport/binder/end2end/fuzzers/client_fuzzer.cc b/test/core/transport/binder/end2end/fuzzers/client_fuzzer.cc index a28618c0725..dc1f3887110 100644 --- a/test/core/transport/binder/end2end/fuzzers/client_fuzzer.cc +++ b/test/core/transport/binder/end2end/fuzzers/client_fuzzer.cc @@ -24,6 +24,7 @@ #include "src/core/lib/config/core_configuration.h" #include "src/core/lib/iomgr/executor.h" #include "src/core/lib/surface/channel.h" +#include "src/core/lib/surface/channel_create.h" #include "src/libfuzzer/libfuzzer_macro.h" #include "test/core/transport/binder/end2end/fuzzers/binder_transport_fuzzer.pb.h" #include "test/core/transport/binder/end2end/fuzzers/fuzzer_utils.h" @@ -59,8 +60,8 @@ DEFINE_PROTO_FUZZER(const binder_transport_fuzzer::Input& input) { .channel_args_preconditioning() .PreconditionChannelArgs(args); auto channel = - grpc_core::Channel::Create("test-target", channel_args, - GRPC_CLIENT_DIRECT_CHANNEL, client_transport) + grpc_core::ChannelCreate("test-target", channel_args, + GRPC_CLIENT_DIRECT_CHANNEL, client_transport) ->release() ->c_ptr(); grpc_channel_args_destroy(args); diff --git a/test/core/transport/binder/end2end/testing_channel_create.cc b/test/core/transport/binder/end2end/testing_channel_create.cc index c9a062b954d..1823498bf26 100644 --- a/test/core/transport/binder/end2end/testing_channel_create.cc +++ b/test/core/transport/binder/end2end/testing_channel_create.cc @@ -23,6 +23,7 @@ #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/config/core_configuration.h" #include "src/core/lib/surface/channel.h" +#include "src/core/lib/surface/channel_create.h" #include "src/core/lib/transport/error_utils.h" namespace grpc_binder { @@ -124,7 +125,7 @@ grpc_channel* grpc_binder_channel_create_for_testing( grpc_error_handle error = grpc_core::Server::FromC(server)->SetupTransport( server_transport, nullptr, server_args, nullptr); GPR_ASSERT(error.ok()); - auto channel = grpc_core::Channel::Create( + auto channel = grpc_core::ChannelCreate( "binder", client_args, GRPC_CLIENT_DIRECT_CHANNEL, client_transport); GPR_ASSERT(channel.ok()); return channel->release()->c_ptr(); diff --git a/test/core/transport/chaotic_good/BUILD b/test/core/transport/chaotic_good/BUILD index f76d7c35041..c288be7ade5 100644 --- a/test/core/transport/chaotic_good/BUILD +++ b/test/core/transport/chaotic_good/BUILD @@ -82,7 +82,10 @@ grpc_cc_test( "absl/status:statusor", "gtest", ], - deps = ["//src/core:chaotic_good_frame"], + deps = [ + "//:channel_create", + "//src/core:chaotic_good_frame", + ], ) grpc_proto_fuzzer( diff --git a/test/cpp/microbenchmarks/fullstack_fixtures.h b/test/cpp/microbenchmarks/fullstack_fixtures.h index ca3b49e0706..e2f9cfefc6d 100644 --- a/test/cpp/microbenchmarks/fullstack_fixtures.h +++ b/test/cpp/microbenchmarks/fullstack_fixtures.h @@ -37,6 +37,7 @@ #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/iomgr/tcp_posix.h" #include "src/core/lib/surface/channel.h" +#include "src/core/lib/surface/channel_create.h" #include "src/core/lib/surface/completion_queue.h" #include "src/core/lib/surface/server.h" #include "src/cpp/client/create_channel_internal.h" @@ -206,8 +207,8 @@ class EndpointPairFixture : public BaseFixture { grpc_create_chttp2_transport(c_args, endpoints.client, true); GPR_ASSERT(client_transport_); grpc_channel* channel = - grpc_core::Channel::Create( - "target", c_args, GRPC_CLIENT_DIRECT_CHANNEL, client_transport_) + grpc_core::ChannelCreate("target", c_args, GRPC_CLIENT_DIRECT_CHANNEL, + client_transport_) ->release() ->c_ptr(); grpc_chttp2_transport_start_reading(client_transport_, nullptr, nullptr, diff --git a/test/cpp/performance/BUILD b/test/cpp/performance/BUILD index c349db8fc10..15de8b3d73c 100644 --- a/test/cpp/performance/BUILD +++ b/test/cpp/performance/BUILD @@ -29,6 +29,7 @@ grpc_cc_test( "//:gpr", "//:grpc", "//:grpc++", + "//:server", "//src/core:channel_args", "//src/proto/grpc/testing:echo_proto", "//test/core/event_engine:event_engine_test_utils", diff --git a/test/cpp/performance/writes_per_rpc_test.cc b/test/cpp/performance/writes_per_rpc_test.cc index 54abcebdf08..4b38fbbc258 100644 --- a/test/cpp/performance/writes_per_rpc_test.cc +++ b/test/cpp/performance/writes_per_rpc_test.cc @@ -41,6 +41,7 @@ #include "src/core/lib/iomgr/event_engine_shims/endpoint.h" #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/surface/channel.h" +#include "src/core/lib/surface/channel_create.h" #include "src/core/lib/surface/server.h" #include "src/cpp/client/create_channel_internal.h" #include "src/proto/grpc/testing/echo.grpc.pb.h" @@ -144,8 +145,8 @@ class InProcessCHTTP2 { /*is_client=*/true); GPR_ASSERT(transport); grpc_channel* channel = - grpc_core::Channel::Create("target", args, GRPC_CLIENT_DIRECT_CHANNEL, - transport) + grpc_core::ChannelCreate("target", args, GRPC_CLIENT_DIRECT_CHANNEL, + transport) ->release() ->c_ptr(); grpc_chttp2_transport_start_reading(transport, nullptr, nullptr, nullptr); diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index eae188277ca..ae04b133f2f 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -1083,7 +1083,6 @@ include/grpcpp/version_info.h \ include/grpcpp/xds_server_builder.h \ src/core/client_channel/backup_poller.cc \ src/core/client_channel/backup_poller.h \ -src/core/client_channel/channel_connectivity.cc \ src/core/client_channel/client_channel_channelz.cc \ src/core/client_channel/client_channel_channelz.h \ src/core/client_channel/client_channel_factory.cc \ @@ -2761,8 +2760,6 @@ src/core/lib/slice/slice_string_helpers.cc \ src/core/lib/slice/slice_string_helpers.h \ src/core/lib/surface/api_trace.cc \ src/core/lib/surface/api_trace.h \ -src/core/lib/surface/builtins.cc \ -src/core/lib/surface/builtins.h \ src/core/lib/surface/byte_buffer.cc \ src/core/lib/surface/byte_buffer_reader.cc \ src/core/lib/surface/call.cc \ @@ -2773,9 +2770,10 @@ src/core/lib/surface/call_test_only.h \ src/core/lib/surface/call_trace.h \ src/core/lib/surface/channel.cc \ src/core/lib/surface/channel.h \ +src/core/lib/surface/channel_create.cc \ +src/core/lib/surface/channel_create.h \ src/core/lib/surface/channel_init.cc \ src/core/lib/surface/channel_init.h \ -src/core/lib/surface/channel_ping.cc \ src/core/lib/surface/channel_stack_type.cc \ src/core/lib/surface/channel_stack_type.h \ src/core/lib/surface/completion_queue.cc \ @@ -2790,9 +2788,12 @@ src/core/lib/surface/init_internally.cc \ src/core/lib/surface/init_internally.h \ src/core/lib/surface/lame_client.cc \ src/core/lib/surface/lame_client.h \ +src/core/lib/surface/legacy_channel.cc \ +src/core/lib/surface/legacy_channel.h \ src/core/lib/surface/metadata_array.cc \ src/core/lib/surface/server.cc \ src/core/lib/surface/server.h \ +src/core/lib/surface/server_interface.h \ src/core/lib/surface/validate_metadata.cc \ src/core/lib/surface/validate_metadata.h \ src/core/lib/surface/version.cc \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 145fc7f8027..5990bd20e58 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -887,7 +887,6 @@ src/core/README.md \ src/core/client_channel/README.md \ src/core/client_channel/backup_poller.cc \ src/core/client_channel/backup_poller.h \ -src/core/client_channel/channel_connectivity.cc \ src/core/client_channel/client_channel_channelz.cc \ src/core/client_channel/client_channel_channelz.h \ src/core/client_channel/client_channel_factory.cc \ @@ -2537,8 +2536,6 @@ src/core/lib/slice/slice_string_helpers.h \ src/core/lib/surface/README.md \ src/core/lib/surface/api_trace.cc \ src/core/lib/surface/api_trace.h \ -src/core/lib/surface/builtins.cc \ -src/core/lib/surface/builtins.h \ src/core/lib/surface/byte_buffer.cc \ src/core/lib/surface/byte_buffer_reader.cc \ src/core/lib/surface/call.cc \ @@ -2549,9 +2546,10 @@ src/core/lib/surface/call_test_only.h \ src/core/lib/surface/call_trace.h \ src/core/lib/surface/channel.cc \ src/core/lib/surface/channel.h \ +src/core/lib/surface/channel_create.cc \ +src/core/lib/surface/channel_create.h \ src/core/lib/surface/channel_init.cc \ src/core/lib/surface/channel_init.h \ -src/core/lib/surface/channel_ping.cc \ src/core/lib/surface/channel_stack_type.cc \ src/core/lib/surface/channel_stack_type.h \ src/core/lib/surface/completion_queue.cc \ @@ -2566,9 +2564,12 @@ src/core/lib/surface/init_internally.cc \ src/core/lib/surface/init_internally.h \ src/core/lib/surface/lame_client.cc \ src/core/lib/surface/lame_client.h \ +src/core/lib/surface/legacy_channel.cc \ +src/core/lib/surface/legacy_channel.h \ src/core/lib/surface/metadata_array.cc \ src/core/lib/surface/server.cc \ src/core/lib/surface/server.h \ +src/core/lib/surface/server_interface.h \ src/core/lib/surface/validate_metadata.cc \ src/core/lib/surface/validate_metadata.h \ src/core/lib/surface/version.cc \ diff --git a/tools/run_tests/sanity/core_banned_functions.py b/tools/run_tests/sanity/core_banned_functions.py index 5340cc87ce1..7804dd0fe52 100755 --- a/tools/run_tests/sanity/core_banned_functions.py +++ b/tools/run_tests/sanity/core_banned_functions.py @@ -55,6 +55,7 @@ BANNED_EXCEPT = { "grpc_call_cancel(": ["src/core/lib/surface/call.cc"], "grpc_channel_destroy(": [ "src/core/lib/surface/channel.cc", + "src/core/lib/surface/legacy_channel.cc", "src/core/tsi/alts/handshaker/alts_shared_resource.cc", ], "grpc_closure_create(": [ From c43f1a63b0403264f0b37b136bcc082f1d94bc63 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Mon, 4 Mar 2024 11:56:02 -0800 Subject: [PATCH 02/52] [RLS] implement non-per-call metrics (#36001) Closes #36001 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/36001 from markdroth:non_per_call_metrics_rls b713980f21e9bea7a0b4d53a8ce3adfa2f8ae1a9 PiperOrigin-RevId: 612532005 --- CMakeLists.txt | 4 +- Makefile | 2 + Package.swift | 3 + build_autogenerated.yaml | 12 +- config.m4 | 1 + config.w32 | 1 + gRPC-C++.podspec | 4 + gRPC-Core.podspec | 5 + grpc.gemspec | 3 + grpc.gyp | 2 + package.xml | 3 + src/core/BUILD | 9 + src/core/lib/channel/metrics.cc | 145 ++++++ src/core/lib/channel/metrics.h | 137 +++++- src/core/load_balancing/rls/rls.cc | 206 +++++++-- src/core/load_balancing/rls/rls.h | 26 ++ src/python/grpcio/grpc_core_dependencies.py | 1 + test/core/channel/metrics_test.cc | 486 ++++++++++++++++++++ test/core/util/fake_stats_plugin.cc | 38 ++ test/core/util/fake_stats_plugin.h | 226 ++++++++- test/cpp/end2end/BUILD | 1 + test/cpp/end2end/rls_end2end_test.cc | 324 ++++++++++++- tools/doxygen/Doxyfile.c++.internal | 3 + tools/doxygen/Doxyfile.core.internal | 3 + 24 files changed, 1578 insertions(+), 67 deletions(-) create mode 100644 src/core/load_balancing/rls/rls.h diff --git a/CMakeLists.txt b/CMakeLists.txt index d37c313418e..cf8697a597f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2278,6 +2278,7 @@ add_library(grpc src/core/lib/gprpp/status_helper.cc src/core/lib/gprpp/time.cc src/core/lib/gprpp/time_averaged_stats.cc + src/core/lib/gprpp/uuid_v4.cc src/core/lib/gprpp/validation_errors.cc src/core/lib/gprpp/windows/directory_reader.cc src/core/lib/gprpp/work_serializer.cc @@ -3046,6 +3047,7 @@ add_library(grpc_unsecure src/core/lib/gprpp/status_helper.cc src/core/lib/gprpp/time.cc src/core/lib/gprpp/time_averaged_stats.cc + src/core/lib/gprpp/uuid_v4.cc src/core/lib/gprpp/validation_errors.cc src/core/lib/gprpp/work_serializer.cc src/core/lib/handshaker/proxy_mapper_registry.cc @@ -25839,6 +25841,7 @@ add_executable(rls_end2end_test ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/orca_load_report.grpc.pb.cc ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/orca_load_report.pb.h ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/orca_load_report.grpc.pb.h + test/core/util/fake_stats_plugin.cc test/core/util/test_lb_policies.cc test/cpp/end2end/rls_end2end_test.cc test/cpp/end2end/rls_server.cc @@ -31537,7 +31540,6 @@ endif() if(gRPC_BUILD_TESTS) add_executable(uuid_v4_test - src/core/lib/gprpp/uuid_v4.cc test/core/gprpp/uuid_v4_test.cc ) if(WIN32 AND MSVC) diff --git a/Makefile b/Makefile index 77c838096dd..2fd360f7202 100644 --- a/Makefile +++ b/Makefile @@ -1459,6 +1459,7 @@ LIBGRPC_SRC = \ src/core/lib/gprpp/status_helper.cc \ src/core/lib/gprpp/time.cc \ src/core/lib/gprpp/time_averaged_stats.cc \ + src/core/lib/gprpp/uuid_v4.cc \ src/core/lib/gprpp/validation_errors.cc \ src/core/lib/gprpp/windows/directory_reader.cc \ src/core/lib/gprpp/work_serializer.cc \ @@ -2060,6 +2061,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/gprpp/status_helper.cc \ src/core/lib/gprpp/time.cc \ src/core/lib/gprpp/time_averaged_stats.cc \ + src/core/lib/gprpp/uuid_v4.cc \ src/core/lib/gprpp/validation_errors.cc \ src/core/lib/gprpp/work_serializer.cc \ src/core/lib/handshaker/proxy_mapper_registry.cc \ diff --git a/Package.swift b/Package.swift index 6f480c481de..64d6d866bf7 100644 --- a/Package.swift +++ b/Package.swift @@ -1393,6 +1393,8 @@ let package = Package( "src/core/lib/gprpp/time_util.h", "src/core/lib/gprpp/type_list.h", "src/core/lib/gprpp/unique_type_name.h", + "src/core/lib/gprpp/uuid_v4.cc", + "src/core/lib/gprpp/uuid_v4.h", "src/core/lib/gprpp/validation_errors.cc", "src/core/lib/gprpp/validation_errors.h", "src/core/lib/gprpp/windows/directory_reader.cc", @@ -1888,6 +1890,7 @@ let package = Package( "src/core/load_balancing/ring_hash/ring_hash.cc", "src/core/load_balancing/ring_hash/ring_hash.h", "src/core/load_balancing/rls/rls.cc", + "src/core/load_balancing/rls/rls.h", "src/core/load_balancing/round_robin/round_robin.cc", "src/core/load_balancing/subchannel_interface.h", "src/core/load_balancing/subchannel_list.h", diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index 073ea701ec2..daafa1d41bd 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -926,6 +926,7 @@ libs: - src/core/lib/gprpp/time_averaged_stats.h - src/core/lib/gprpp/type_list.h - src/core/lib/gprpp/unique_type_name.h + - src/core/lib/gprpp/uuid_v4.h - src/core/lib/gprpp/validation_errors.h - src/core/lib/gprpp/work_serializer.h - src/core/lib/gprpp/xxhash_inline.h @@ -1178,6 +1179,7 @@ libs: - src/core/load_balancing/outlier_detection/outlier_detection.h - src/core/load_balancing/pick_first/pick_first.h - src/core/load_balancing/ring_hash/ring_hash.h + - src/core/load_balancing/rls/rls.h - src/core/load_balancing/subchannel_interface.h - src/core/load_balancing/subchannel_list.h - src/core/load_balancing/weighted_round_robin/static_stride_scheduler.h @@ -1738,6 +1740,7 @@ libs: - src/core/lib/gprpp/status_helper.cc - src/core/lib/gprpp/time.cc - src/core/lib/gprpp/time_averaged_stats.cc + - src/core/lib/gprpp/uuid_v4.cc - src/core/lib/gprpp/validation_errors.cc - src/core/lib/gprpp/windows/directory_reader.cc - src/core/lib/gprpp/work_serializer.cc @@ -2428,6 +2431,7 @@ libs: - src/core/lib/gprpp/time_averaged_stats.h - src/core/lib/gprpp/type_list.h - src/core/lib/gprpp/unique_type_name.h + - src/core/lib/gprpp/uuid_v4.h - src/core/lib/gprpp/validation_errors.h - src/core/lib/gprpp/work_serializer.h - src/core/lib/handshaker/proxy_mapper.h @@ -2644,6 +2648,7 @@ libs: - src/core/load_balancing/oob_backend_metric_internal.h - src/core/load_balancing/outlier_detection/outlier_detection.h - src/core/load_balancing/pick_first/pick_first.h + - src/core/load_balancing/rls/rls.h - src/core/load_balancing/subchannel_interface.h - src/core/load_balancing/subchannel_list.h - src/core/load_balancing/weighted_round_robin/static_stride_scheduler.h @@ -2863,6 +2868,7 @@ libs: - src/core/lib/gprpp/status_helper.cc - src/core/lib/gprpp/time.cc - src/core/lib/gprpp/time_averaged_stats.cc + - src/core/lib/gprpp/uuid_v4.cc - src/core/lib/gprpp/validation_errors.cc - src/core/lib/gprpp/work_serializer.cc - src/core/lib/handshaker/proxy_mapper_registry.cc @@ -16869,6 +16875,7 @@ targets: run: false language: c++ headers: + - test/core/util/fake_stats_plugin.h - test/core/util/test_lb_policies.h - test/cpp/end2end/counted_service.h - test/cpp/end2end/rls_server.h @@ -16880,6 +16887,7 @@ targets: - src/proto/grpc/testing/echo_messages.proto - src/proto/grpc/testing/simple_messages.proto - src/proto/grpc/testing/xds/v3/orca_load_report.proto + - test/core/util/fake_stats_plugin.cc - test/core/util/test_lb_policies.cc - test/cpp/end2end/rls_end2end_test.cc - test/cpp/end2end/rls_server.cc @@ -19503,10 +19511,8 @@ targets: gtest: true build: test language: c++ - headers: - - src/core/lib/gprpp/uuid_v4.h + headers: [] src: - - src/core/lib/gprpp/uuid_v4.cc - test/core/gprpp/uuid_v4_test.cc deps: - gtest diff --git a/config.m4 b/config.m4 index 950fdbb4632..4600f743302 100644 --- a/config.m4 +++ b/config.m4 @@ -584,6 +584,7 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/gprpp/time.cc \ src/core/lib/gprpp/time_averaged_stats.cc \ src/core/lib/gprpp/time_util.cc \ + src/core/lib/gprpp/uuid_v4.cc \ src/core/lib/gprpp/validation_errors.cc \ src/core/lib/gprpp/windows/directory_reader.cc \ src/core/lib/gprpp/windows/env.cc \ diff --git a/config.w32 b/config.w32 index 59efe403184..dcf40dbb7d3 100644 --- a/config.w32 +++ b/config.w32 @@ -549,6 +549,7 @@ if (PHP_GRPC != "no") { "src\\core\\lib\\gprpp\\time.cc " + "src\\core\\lib\\gprpp\\time_averaged_stats.cc " + "src\\core\\lib\\gprpp\\time_util.cc " + + "src\\core\\lib\\gprpp\\uuid_v4.cc " + "src\\core\\lib\\gprpp\\validation_errors.cc " + "src\\core\\lib\\gprpp\\windows\\directory_reader.cc " + "src\\core\\lib\\gprpp\\windows\\env.cc " + diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index a03916faa25..84a7bfef691 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -1032,6 +1032,7 @@ Pod::Spec.new do |s| 'src/core/lib/gprpp/time_util.h', 'src/core/lib/gprpp/type_list.h', 'src/core/lib/gprpp/unique_type_name.h', + 'src/core/lib/gprpp/uuid_v4.h', 'src/core/lib/gprpp/validation_errors.h', 'src/core/lib/gprpp/work_serializer.h', 'src/core/lib/gprpp/xxhash_inline.h', @@ -1284,6 +1285,7 @@ Pod::Spec.new do |s| 'src/core/load_balancing/outlier_detection/outlier_detection.h', 'src/core/load_balancing/pick_first/pick_first.h', 'src/core/load_balancing/ring_hash/ring_hash.h', + 'src/core/load_balancing/rls/rls.h', 'src/core/load_balancing/subchannel_interface.h', 'src/core/load_balancing/subchannel_list.h', 'src/core/load_balancing/weighted_round_robin/static_stride_scheduler.h', @@ -2293,6 +2295,7 @@ Pod::Spec.new do |s| 'src/core/lib/gprpp/time_util.h', 'src/core/lib/gprpp/type_list.h', 'src/core/lib/gprpp/unique_type_name.h', + 'src/core/lib/gprpp/uuid_v4.h', 'src/core/lib/gprpp/validation_errors.h', 'src/core/lib/gprpp/work_serializer.h', 'src/core/lib/gprpp/xxhash_inline.h', @@ -2545,6 +2548,7 @@ Pod::Spec.new do |s| 'src/core/load_balancing/outlier_detection/outlier_detection.h', 'src/core/load_balancing/pick_first/pick_first.h', 'src/core/load_balancing/ring_hash/ring_hash.h', + 'src/core/load_balancing/rls/rls.h', 'src/core/load_balancing/subchannel_interface.h', 'src/core/load_balancing/subchannel_list.h', 'src/core/load_balancing/weighted_round_robin/static_stride_scheduler.h', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index d301e36c83b..35adc20a350 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -1507,6 +1507,8 @@ Pod::Spec.new do |s| 'src/core/lib/gprpp/time_util.h', 'src/core/lib/gprpp/type_list.h', 'src/core/lib/gprpp/unique_type_name.h', + 'src/core/lib/gprpp/uuid_v4.cc', + 'src/core/lib/gprpp/uuid_v4.h', 'src/core/lib/gprpp/validation_errors.cc', 'src/core/lib/gprpp/validation_errors.h', 'src/core/lib/gprpp/windows/directory_reader.cc', @@ -1998,6 +2000,7 @@ Pod::Spec.new do |s| 'src/core/load_balancing/ring_hash/ring_hash.cc', 'src/core/load_balancing/ring_hash/ring_hash.h', 'src/core/load_balancing/rls/rls.cc', + 'src/core/load_balancing/rls/rls.h', 'src/core/load_balancing/round_robin/round_robin.cc', 'src/core/load_balancing/subchannel_interface.h', 'src/core/load_balancing/subchannel_list.h', @@ -3073,6 +3076,7 @@ Pod::Spec.new do |s| 'src/core/lib/gprpp/time_util.h', 'src/core/lib/gprpp/type_list.h', 'src/core/lib/gprpp/unique_type_name.h', + 'src/core/lib/gprpp/uuid_v4.h', 'src/core/lib/gprpp/validation_errors.h', 'src/core/lib/gprpp/work_serializer.h', 'src/core/lib/gprpp/xxhash_inline.h', @@ -3325,6 +3329,7 @@ Pod::Spec.new do |s| 'src/core/load_balancing/outlier_detection/outlier_detection.h', 'src/core/load_balancing/pick_first/pick_first.h', 'src/core/load_balancing/ring_hash/ring_hash.h', + 'src/core/load_balancing/rls/rls.h', 'src/core/load_balancing/subchannel_interface.h', 'src/core/load_balancing/subchannel_list.h', 'src/core/load_balancing/weighted_round_robin/static_stride_scheduler.h', diff --git a/grpc.gemspec b/grpc.gemspec index f31e053b11d..eaed281aa80 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -1399,6 +1399,8 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/gprpp/time_util.h ) s.files += %w( src/core/lib/gprpp/type_list.h ) s.files += %w( src/core/lib/gprpp/unique_type_name.h ) + s.files += %w( src/core/lib/gprpp/uuid_v4.cc ) + s.files += %w( src/core/lib/gprpp/uuid_v4.h ) s.files += %w( src/core/lib/gprpp/validation_errors.cc ) s.files += %w( src/core/lib/gprpp/validation_errors.h ) s.files += %w( src/core/lib/gprpp/windows/directory_reader.cc ) @@ -1890,6 +1892,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/load_balancing/ring_hash/ring_hash.cc ) s.files += %w( src/core/load_balancing/ring_hash/ring_hash.h ) s.files += %w( src/core/load_balancing/rls/rls.cc ) + s.files += %w( src/core/load_balancing/rls/rls.h ) s.files += %w( src/core/load_balancing/round_robin/round_robin.cc ) s.files += %w( src/core/load_balancing/subchannel_interface.h ) s.files += %w( src/core/load_balancing/subchannel_list.h ) diff --git a/grpc.gyp b/grpc.gyp index fcffddc3163..bb844663ee0 100644 --- a/grpc.gyp +++ b/grpc.gyp @@ -774,6 +774,7 @@ 'src/core/lib/gprpp/status_helper.cc', 'src/core/lib/gprpp/time.cc', 'src/core/lib/gprpp/time_averaged_stats.cc', + 'src/core/lib/gprpp/uuid_v4.cc', 'src/core/lib/gprpp/validation_errors.cc', 'src/core/lib/gprpp/windows/directory_reader.cc', 'src/core/lib/gprpp/work_serializer.cc', @@ -1316,6 +1317,7 @@ 'src/core/lib/gprpp/status_helper.cc', 'src/core/lib/gprpp/time.cc', 'src/core/lib/gprpp/time_averaged_stats.cc', + 'src/core/lib/gprpp/uuid_v4.cc', 'src/core/lib/gprpp/validation_errors.cc', 'src/core/lib/gprpp/work_serializer.cc', 'src/core/lib/handshaker/proxy_mapper_registry.cc', diff --git a/package.xml b/package.xml index 42ca1197a08..fa35a2c4033 100644 --- a/package.xml +++ b/package.xml @@ -1381,6 +1381,8 @@ + + @@ -1872,6 +1874,7 @@ + diff --git a/src/core/BUILD b/src/core/BUILD index 185aa98a554..3f80297081c 100644 --- a/src/core/BUILD +++ b/src/core/BUILD @@ -4802,9 +4802,13 @@ grpc_cc_library( srcs = [ "load_balancing/rls/rls.cc", ], + hdrs = [ + "load_balancing/rls/rls.h", + ], external_deps = [ "absl/base:core_headers", "absl/hash", + "absl/random", "absl/status", "absl/status:statusor", "absl/strings", @@ -4830,11 +4834,14 @@ grpc_cc_library( "lb_policy", "lb_policy_factory", "lb_policy_registry", + "match", + "metrics", "pollset_set", "slice", "slice_refcount", "status_helper", "time", + "uuid_v4", "validation_errors", "//:backoff", "//:channel", @@ -7560,6 +7567,7 @@ grpc_cc_library( ], external_deps = [ "absl/container:flat_hash_map", + "absl/functional:any_invocable", "absl/functional:function_ref", "absl/strings", "absl/types:span", @@ -7568,6 +7576,7 @@ grpc_cc_library( deps = [ "channel_args", "no_destruct", + "time", "//:gpr", ], ) diff --git a/src/core/lib/channel/metrics.cc b/src/core/lib/channel/metrics.cc index 36e17d2afc9..5a038db187b 100644 --- a/src/core/lib/channel/metrics.cc +++ b/src/core/lib/channel/metrics.cc @@ -149,6 +149,122 @@ GlobalInstrumentsRegistry::RegisterDoubleHistogram( return handle; } +GlobalInstrumentsRegistry::GlobalInt64GaugeHandle +GlobalInstrumentsRegistry::RegisterInt64Gauge( + absl::string_view name, absl::string_view description, + absl::string_view unit, absl::Span label_keys, + absl::Span optional_label_keys, + bool enable_by_default) { + auto& instruments = GetInstrumentList(); + if (instruments.find(name) != instruments.end()) { + Crash(absl::StrFormat("Metric name %s has already been registered.", name)); + } + uint32_t index = instruments.size(); + GPR_ASSERT(index < std::numeric_limits::max()); + GlobalInstrumentDescriptor descriptor; + descriptor.value_type = ValueType::kInt64; + descriptor.instrument_type = InstrumentType::kGauge; + descriptor.index = index; + descriptor.enable_by_default = enable_by_default; + descriptor.name = name; + descriptor.description = description; + descriptor.unit = unit; + descriptor.label_keys = {label_keys.begin(), label_keys.end()}; + descriptor.optional_label_keys = {optional_label_keys.begin(), + optional_label_keys.end()}; + instruments.emplace(name, std::move(descriptor)); + GlobalInt64GaugeHandle handle; + handle.index = index; + return handle; +} + +GlobalInstrumentsRegistry::GlobalDoubleGaugeHandle +GlobalInstrumentsRegistry::RegisterDoubleGauge( + absl::string_view name, absl::string_view description, + absl::string_view unit, absl::Span label_keys, + absl::Span optional_label_keys, + bool enable_by_default) { + auto& instruments = GetInstrumentList(); + if (instruments.find(name) != instruments.end()) { + Crash(absl::StrFormat("Metric name %s has already been registered.", name)); + } + uint32_t index = instruments.size(); + GPR_ASSERT(index < std::numeric_limits::max()); + GlobalInstrumentDescriptor descriptor; + descriptor.value_type = ValueType::kDouble; + descriptor.instrument_type = InstrumentType::kGauge; + descriptor.index = index; + descriptor.enable_by_default = enable_by_default; + descriptor.name = name; + descriptor.description = description; + descriptor.unit = unit; + descriptor.label_keys = {label_keys.begin(), label_keys.end()}; + descriptor.optional_label_keys = {optional_label_keys.begin(), + optional_label_keys.end()}; + instruments.emplace(name, std::move(descriptor)); + GlobalDoubleGaugeHandle handle; + handle.index = index; + return handle; +} + +GlobalInstrumentsRegistry::GlobalCallbackInt64GaugeHandle +GlobalInstrumentsRegistry::RegisterCallbackInt64Gauge( + absl::string_view name, absl::string_view description, + absl::string_view unit, absl::Span label_keys, + absl::Span optional_label_keys, + bool enable_by_default) { + auto& instruments = GetInstrumentList(); + if (instruments.find(name) != instruments.end()) { + Crash(absl::StrFormat("Metric name %s has already been registered.", name)); + } + uint32_t index = instruments.size(); + GPR_ASSERT(index < std::numeric_limits::max()); + GlobalInstrumentDescriptor descriptor; + descriptor.value_type = ValueType::kInt64; + descriptor.instrument_type = InstrumentType::kCallbackGauge; + descriptor.index = index; + descriptor.enable_by_default = enable_by_default; + descriptor.name = name; + descriptor.description = description; + descriptor.unit = unit; + descriptor.label_keys = {label_keys.begin(), label_keys.end()}; + descriptor.optional_label_keys = {optional_label_keys.begin(), + optional_label_keys.end()}; + instruments.emplace(name, std::move(descriptor)); + GlobalCallbackInt64GaugeHandle handle; + handle.index = index; + return handle; +} + +GlobalInstrumentsRegistry::GlobalCallbackDoubleGaugeHandle +GlobalInstrumentsRegistry::RegisterCallbackDoubleGauge( + absl::string_view name, absl::string_view description, + absl::string_view unit, absl::Span label_keys, + absl::Span optional_label_keys, + bool enable_by_default) { + auto& instruments = GetInstrumentList(); + if (instruments.find(name) != instruments.end()) { + Crash(absl::StrFormat("Metric name %s has already been registered.", name)); + } + uint32_t index = instruments.size(); + GPR_ASSERT(index < std::numeric_limits::max()); + GlobalInstrumentDescriptor descriptor; + descriptor.value_type = ValueType::kDouble; + descriptor.instrument_type = InstrumentType::kCallbackGauge; + descriptor.index = index; + descriptor.enable_by_default = enable_by_default; + descriptor.name = name; + descriptor.description = description; + descriptor.unit = unit; + descriptor.label_keys = {label_keys.begin(), label_keys.end()}; + descriptor.optional_label_keys = {optional_label_keys.begin(), + optional_label_keys.end()}; + instruments.emplace(name, std::move(descriptor)); + GlobalCallbackDoubleGaugeHandle handle; + handle.index = index; + return handle; +} + void GlobalInstrumentsRegistry::ForEach( absl::FunctionRef f) { for (const auto& instrument : GetInstrumentList()) { @@ -156,6 +272,35 @@ void GlobalInstrumentsRegistry::ForEach( } } +RegisteredMetricCallback::RegisteredMetricCallback( + GlobalStatsPluginRegistry::StatsPluginGroup& stats_plugin_group, + absl::AnyInvocable callback, + std::vector metrics, + Duration min_interval) + : stats_plugin_group_(stats_plugin_group), + callback_(std::move(callback)), + metrics_(std::move(metrics)), + min_interval_(min_interval) { + for (auto& plugin : stats_plugin_group_.plugins_) { + plugin->AddCallback(this); + } +} + +RegisteredMetricCallback::~RegisteredMetricCallback() { + for (auto& plugin : stats_plugin_group_.plugins_) { + plugin->RemoveCallback(this); + } +} + +std::unique_ptr +GlobalStatsPluginRegistry::StatsPluginGroup::RegisterCallback( + absl::AnyInvocable callback, + std::vector metrics, + Duration min_interval) { + return std::make_unique( + *this, std::move(callback), std::move(metrics), min_interval); +} + NoDestruct GlobalStatsPluginRegistry::mutex_; NoDestruct>> GlobalStatsPluginRegistry::plugins_; diff --git a/src/core/lib/channel/metrics.h b/src/core/lib/channel/metrics.h index 51803e7c4fa..a93f0037cb3 100644 --- a/src/core/lib/channel/metrics.h +++ b/src/core/lib/channel/metrics.h @@ -22,6 +22,7 @@ #include #include "absl/container/flat_hash_map.h" +#include "absl/functional/any_invocable.h" #include "absl/functional/function_ref.h" #include "absl/strings/string_view.h" #include "absl/types/span.h" @@ -31,20 +32,22 @@ #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/gprpp/no_destruct.h" #include "src/core/lib/gprpp/sync.h" +#include "src/core/lib/gprpp/time.h" namespace grpc_core { constexpr absl::string_view kMetricLabelTarget = "grpc.target"; -// A global registry of instruments(metrics). This API is designed to be used to -// register instruments (Counter and Histogram) as part of program startup, -// before the execution of the main function (during dynamic initialization -// time). Using this API after the main function begins may result into missing -// instruments. This API is thread-unsafe. +// A global registry of instruments(metrics). This API is designed to be used +// to register instruments (Counter, Histogram, and Gauge) as part of program +// startup, before the execution of the main function (during dynamic +// initialization time). Using this API after the main function begins may +// result into missing instruments. This API is thread-unsafe. class GlobalInstrumentsRegistry { public: enum class ValueType { kUndefined, + kInt64, kUInt64, kDouble, }; @@ -52,6 +55,8 @@ class GlobalInstrumentsRegistry { kUndefined, kCounter, kHistogram, + kGauge, + kCallbackGauge, }; struct GlobalInstrumentDescriptor { ValueType value_type; @@ -75,6 +80,11 @@ class GlobalInstrumentsRegistry { struct GlobalDoubleCounterHandle : public GlobalHandle {}; struct GlobalUInt64HistogramHandle : public GlobalHandle {}; struct GlobalDoubleHistogramHandle : public GlobalHandle {}; + struct GlobalInt64GaugeHandle : public GlobalHandle {}; + struct GlobalDoubleGaugeHandle : public GlobalHandle {}; + struct GlobalCallbackHandle : public GlobalHandle {}; + struct GlobalCallbackInt64GaugeHandle : public GlobalCallbackHandle {}; + struct GlobalCallbackDoubleGaugeHandle : public GlobalCallbackHandle {}; // Creates instrument in the GlobalInstrumentsRegistry. static GlobalUInt64CounterHandle RegisterUInt64Counter( @@ -97,6 +107,27 @@ class GlobalInstrumentsRegistry { absl::string_view unit, absl::Span label_keys, absl::Span optional_label_keys, bool enable_by_default); + static GlobalInt64GaugeHandle RegisterInt64Gauge( + absl::string_view name, absl::string_view description, + absl::string_view unit, absl::Span label_keys, + absl::Span optional_label_keys, + bool enable_by_default); + static GlobalDoubleGaugeHandle RegisterDoubleGauge( + absl::string_view name, absl::string_view description, + absl::string_view unit, absl::Span label_keys, + absl::Span optional_label_keys, + bool enable_by_default); + static GlobalCallbackInt64GaugeHandle RegisterCallbackInt64Gauge( + absl::string_view name, absl::string_view description, + absl::string_view unit, absl::Span label_keys, + absl::Span optional_label_keys, + bool enable_by_default); + static GlobalCallbackDoubleGaugeHandle RegisterCallbackDoubleGauge( + absl::string_view name, absl::string_view description, + absl::string_view unit, absl::Span label_keys, + absl::Span optional_label_keys, + bool enable_by_default); + static void ForEach( absl::FunctionRef f); @@ -110,6 +141,24 @@ class GlobalInstrumentsRegistry { GetInstrumentList(); }; +// An interface for implementing callback-style metrics. +// To be implemented by stats plugins. +class CallbackMetricReporter { + public: + virtual ~CallbackMetricReporter() = default; + + virtual void Report( + GlobalInstrumentsRegistry::GlobalCallbackInt64GaugeHandle handle, + int64_t value, absl::Span label_values, + absl::Span optional_values) = 0; + virtual void Report( + GlobalInstrumentsRegistry::GlobalCallbackDoubleGaugeHandle handle, + double value, absl::Span label_values, + absl::Span optional_values) = 0; +}; + +class RegisteredMetricCallback; + // The StatsPlugin interface. class StatsPlugin { public: @@ -147,6 +196,22 @@ class StatsPlugin { GlobalInstrumentsRegistry::GlobalDoubleHistogramHandle handle, double value, absl::Span label_values, absl::Span optional_values) = 0; + virtual void SetGauge( + GlobalInstrumentsRegistry::GlobalInt64GaugeHandle handle, int64_t value, + absl::Span label_values, + absl::Span optional_values) = 0; + virtual void SetGauge( + GlobalInstrumentsRegistry::GlobalDoubleGaugeHandle handle, double value, + absl::Span label_values, + absl::Span optional_values) = 0; + + // Adds a callback to be invoked when the stats plugin wants to + // populate the corresponding metrics (see callback->metrics() for list). + virtual void AddCallback(RegisteredMetricCallback* callback) = 0; + // Removes a callback previously added via AddCallback(). The stats + // plugin may not use the callback after this method returns. + virtual void RemoveCallback(RegisteredMetricCallback* callback) = 0; + // TODO(yijiem): Details pending. // std::unique_ptr GetObservableGauge( // absl::string_view name, absl::string_view description, @@ -208,8 +273,39 @@ class GlobalStatsPluginRegistry { plugin->RecordHistogram(handle, value, label_values, optional_values); } } + void SetGauge(GlobalInstrumentsRegistry::GlobalInt64GaugeHandle handle, + int64_t value, + absl::Span label_values, + absl::Span optional_values) { + for (auto& plugin : plugins_) { + plugin->SetGauge(handle, value, label_values, optional_values); + } + } + void SetGauge(GlobalInstrumentsRegistry::GlobalDoubleGaugeHandle handle, + double value, + absl::Span label_values, + absl::Span optional_values) { + for (auto& plugin : plugins_) { + plugin->SetGauge(handle, value, label_values, optional_values); + } + } + + // Registers a callback to be used to populate callback metrics. + // The callback will update the specified metrics. The callback + // will be invoked no more often than min_interval. + // + // The returned object is a handle that allows the caller to control + // the lifetime of the callback; when the returned object is + // destroyed, the callback is de-registered. The returned object + // must not outlive the StatsPluginGroup object that created it. + std::unique_ptr RegisterCallback( + absl::AnyInvocable callback, + std::vector metrics, + Duration min_interval = Duration::Seconds(5)); private: + friend class RegisteredMetricCallback; + std::vector> plugins_; }; @@ -231,6 +327,37 @@ class GlobalStatsPluginRegistry { ABSL_GUARDED_BY(mutex_); }; +// A metric callback that is registered with a stats plugin group. +class RegisteredMetricCallback { + public: + RegisteredMetricCallback( + GlobalStatsPluginRegistry::StatsPluginGroup& stats_plugin_group, + absl::AnyInvocable callback, + std::vector metrics, + Duration min_interval); + + ~RegisteredMetricCallback(); + + // Invokes the callback. The callback will report metric data via reporter. + void Run(CallbackMetricReporter& reporter) { callback_(reporter); } + + // Returns the set of metrics that this callback will modify. + const std::vector& metrics() + const { + return metrics_; + } + + // Returns the minimum interval at which a stats plugin may invoke the + // callback. + Duration min_interval() const { return min_interval_; } + + private: + GlobalStatsPluginRegistry::StatsPluginGroup& stats_plugin_group_; + absl::AnyInvocable callback_; + std::vector metrics_; + Duration min_interval_; +}; + } // namespace grpc_core #endif // GRPC_SRC_CORE_LIB_CHANNEL_METRICS_H diff --git a/src/core/load_balancing/rls/rls.cc b/src/core/load_balancing/rls/rls.cc index 4a90a17cfe7..70a9a5b59dd 100644 --- a/src/core/load_balancing/rls/rls.cc +++ b/src/core/load_balancing/rls/rls.cc @@ -22,6 +22,8 @@ #include +#include "src/core/load_balancing/rls/rls.h" + #include #include #include @@ -41,6 +43,7 @@ #include "absl/base/thread_annotations.h" #include "absl/hash/hash.h" +#include "absl/random/random.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" @@ -68,15 +71,18 @@ #include "src/core/lib/backoff/backoff.h" #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/channel/channelz.h" +#include "src/core/lib/channel/metrics.h" #include "src/core/lib/config/core_configuration.h" #include "src/core/lib/debug/trace.h" #include "src/core/lib/gprpp/debug_location.h" #include "src/core/lib/gprpp/dual_ref_counted.h" +#include "src/core/lib/gprpp/match.h" #include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/gprpp/status_helper.h" #include "src/core/lib/gprpp/sync.h" #include "src/core/lib/gprpp/time.h" +#include "src/core/lib/gprpp/uuid_v4.h" #include "src/core/lib/gprpp/validation_errors.h" #include "src/core/lib/gprpp/work_serializer.h" #include "src/core/lib/iomgr/closure.h" @@ -103,13 +109,66 @@ #include "src/core/resolver/resolver_registry.h" #include "src/proto/grpc/lookup/v1/rls.upb.h" +using ::grpc_event_engine::experimental::EventEngine; + namespace grpc_core { TraceFlag grpc_lb_rls_trace(false, "rls_lb"); namespace { -using ::grpc_event_engine::experimental::EventEngine; +constexpr absl::string_view kMetricLabelRlsServerTarget = + "grpc.lb.rls.server_target"; +constexpr absl::string_view kMetricLabelRlsInstanceUuid = + "grpc.lb.rls.instance_uuid"; +constexpr absl::string_view kMetricRlsDataPlaneTarget = + "grpc.lb.rls.data_plane_target"; +constexpr absl::string_view kMetricLabelPickResult = "grpc.lb.pick_result"; + +const auto kMetricCacheSize = + GlobalInstrumentsRegistry::RegisterCallbackInt64Gauge( + "grpc.lb.rls.cache_size", "EXPERIMENTAL. Size of the RLS cache.", + "By", + {kMetricLabelTarget, kMetricLabelRlsServerTarget, + kMetricLabelRlsInstanceUuid}, + {}, false); + +const auto kMetricCacheEntries = + GlobalInstrumentsRegistry::RegisterCallbackInt64Gauge( + "grpc.lb.rls.cache_entries", + "EXPERIMENTAL. Number of entries in the RLS cache.", "{entry}", + {kMetricLabelTarget, kMetricLabelRlsServerTarget, + kMetricLabelRlsInstanceUuid}, + {}, false); + +const auto kMetricDefaultTargetPicks = + GlobalInstrumentsRegistry::RegisterUInt64Counter( + "grpc.lb.rls.default_target_picks", + "EXPERIMENTAL. Number of LB picks sent to the default target.", + "{pick}", + {kMetricLabelTarget, kMetricLabelRlsServerTarget, + kMetricRlsDataPlaneTarget, kMetricLabelPickResult}, + {}, false); + +const auto kMetricTargetPicks = + GlobalInstrumentsRegistry::RegisterUInt64Counter( + "grpc.lb.rls.target_picks", + "EXPERIMENTAL. Number of LB picks sent to each RLS target. Note that " + "if the default target is also returned by the RLS server, RPCs sent " + "to that target from the cache will be counted in this metric, not " + "in grpc.rls.default_target_picks.", + "{pick}", + {kMetricLabelTarget, kMetricLabelRlsServerTarget, + kMetricRlsDataPlaneTarget, kMetricLabelPickResult}, + {}, false); + +const auto kMetricFailedPicks = + GlobalInstrumentsRegistry::RegisterUInt64Counter( + "grpc.lb.rls.failed_picks", + "EXPERIMENTAL. Number of LB picks failed due to either a failed RLS " + "request or the RLS channel being throttled.", + "{pick}", {kMetricLabelTarget, kMetricLabelRlsServerTarget}, {}, + false); constexpr absl::string_view kRls = "rls_experimental"; const char kGrpc[] = "grpc"; @@ -365,6 +424,10 @@ class RlsLb : public LoadBalancingPolicy { PickResult Pick(PickArgs args) override; private: + PickResult PickFromDefaultTargetOrFail(const char* reason, PickArgs args, + absl::Status status) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(&RlsLb::mu_); + RefCountedPtr lb_policy_; RefCountedPtr config_; RefCountedPtr default_child_policy_; @@ -516,6 +579,9 @@ class RlsLb : public LoadBalancingPolicy { // Shutdown the cache; clean-up and orphan all the stored cache entries. void Shutdown() ABSL_EXCLUSIVE_LOCKS_REQUIRED(&RlsLb::mu_); + void ReportMetricsLocked(CallbackMetricReporter& reporter) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(&RlsLb::mu_); + private: // Shared logic for starting the cleanup timer void StartCleanupTimer() ABSL_EXCLUSIVE_LOCKS_REQUIRED(&RlsLb::mu_); @@ -690,6 +756,12 @@ class RlsLb : public LoadBalancingPolicy { // Updates the picker in the work serializer. void UpdatePickerLocked() ABSL_LOCKS_EXCLUDED(&mu_); + void MaybeExportPickCount( + GlobalInstrumentsRegistry::GlobalUInt64CounterHandle handle, + absl::string_view target, const PickResult& pick_result); + + const std::string instance_uuid_; + // Mutex to guard LB policy state that is accessed by the picker. Mutex mu_; bool is_shutdown_ ABSL_GUARDED_BY(mu_) = false; @@ -712,6 +784,9 @@ class RlsLb : public LoadBalancingPolicy { RefCountedPtr config_; RefCountedPtr default_child_policy_; std::map child_policy_map_; + + // Must be after mu_, so that it is destroyed before mu_. + std::unique_ptr registered_metric_callback_; }; // @@ -992,21 +1067,8 @@ LoadBalancingPolicy::PickResult RlsLb::Picker::Pick(PickArgs args) { // If there is no non-expired data in the cache, then we use the // default target if set, or else we fail the pick. if (entry == nullptr || entry->data_expiration_time() < now) { - if (default_child_policy_ != nullptr) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) { - gpr_log(GPR_INFO, - "[rlslb %p] picker=%p: RLS call throttled; " - "using default target", - lb_policy_.get(), this); - } - return default_child_policy_->Pick(args); - } - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) { - gpr_log(GPR_INFO, - "[rlslb %p] picker=%p: RLS call throttled; failing pick", - lb_policy_.get(), this); - } - return PickResult::Fail( + return PickFromDefaultTargetOrFail( + "RLS call throttled", args, absl::UnavailableError("RLS request throttled")); } } @@ -1028,22 +1090,10 @@ LoadBalancingPolicy::PickResult RlsLb::Picker::Pick(PickArgs args) { // If the entry is in backoff, then use the default target if set, // or else fail the pick. if (entry->backoff_time() >= now) { - if (default_child_policy_ != nullptr) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) { - gpr_log( - GPR_INFO, - "[rlslb %p] picker=%p: RLS call in backoff; using default target", - lb_policy_.get(), this); - } - return default_child_policy_->Pick(args); - } - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) { - gpr_log(GPR_INFO, - "[rlslb %p] picker=%p: RLS call in backoff; failing pick", - lb_policy_.get(), this); - } - return PickResult::Fail(absl::UnavailableError( - absl::StrCat("RLS request failed: ", entry->status().ToString()))); + return PickFromDefaultTargetOrFail( + "RLS call in backoff", args, + absl::UnavailableError(absl::StrCat( + "RLS request failed: ", entry->status().ToString()))); } } // RLS call pending. Queue the pick. @@ -1054,6 +1104,31 @@ LoadBalancingPolicy::PickResult RlsLb::Picker::Pick(PickArgs args) { return PickResult::Queue(); } +LoadBalancingPolicy::PickResult RlsLb::Picker::PickFromDefaultTargetOrFail( + const char* reason, PickArgs args, absl::Status status) { + if (default_child_policy_ != nullptr) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) { + gpr_log(GPR_INFO, "[rlslb %p] picker=%p: %s; using default target", + lb_policy_.get(), this, reason); + } + auto pick_result = default_child_policy_->Pick(args); + lb_policy_->MaybeExportPickCount( + kMetricDefaultTargetPicks, config_->default_target(), pick_result); + return pick_result; + } + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) { + gpr_log(GPR_INFO, "[rlslb %p] picker=%p: %s; failing pick", + lb_policy_.get(), this, reason); + } + auto& stats_plugins = + lb_policy_->channel_control_helper()->GetStatsPluginGroup(); + stats_plugins.AddCounter( + kMetricFailedPicks, 1, + {lb_policy_->channel_control_helper()->GetTarget(), + config_->lookup_service()}, {}); + return PickResult::Fail(std::move(status)); +} + // // RlsLb::Cache::Entry::BackoffTimer // @@ -1195,7 +1270,10 @@ LoadBalancingPolicy::PickResult RlsLb::Cache::Entry::Pick(PickArgs args) { strcpy(copied_header_data, header_data_.c_str()); args.initial_metadata->Add(kRlsHeaderKey, copied_header_data); } - return child_policy_wrapper->Pick(args); + auto pick_result = child_policy_wrapper->Pick(args); + lb_policy_->MaybeExportPickCount( + kMetricTargetPicks, child_policy_wrapper->target(), pick_result); + return pick_result; } void RlsLb::Cache::Entry::ResetBackoff() { @@ -1374,6 +1452,19 @@ void RlsLb::Cache::Shutdown() { cleanup_timer_handle_.reset(); } +void RlsLb::Cache::ReportMetricsLocked(CallbackMetricReporter& reporter) { + reporter.Report(kMetricCacheSize, size_, + {lb_policy_->channel_control_helper()->GetTarget(), + lb_policy_->config_->lookup_service(), + lb_policy_->instance_uuid_}, + {}); + reporter.Report(kMetricCacheEntries, map_.size(), + {lb_policy_->channel_control_helper()->GetTarget(), + lb_policy_->config_->lookup_service(), + lb_policy_->instance_uuid_}, + {}); +} + void RlsLb::Cache::StartCleanupTimer() { cleanup_timer_handle_ = lb_policy_->channel_control_helper()->GetEventEngine()->RunAfter( @@ -1846,7 +1937,27 @@ RlsLb::ResponseInfo RlsLb::RlsRequest::ParseResponseProto() { // RlsLb // -RlsLb::RlsLb(Args args) : LoadBalancingPolicy(std::move(args)), cache_(this) { +std::string GenerateUUID() { + absl::uniform_int_distribution distribution; + absl::BitGen bitgen; + uint64_t hi = distribution(bitgen); + uint64_t lo = distribution(bitgen); + return GenerateUUIDv4(hi, lo); +} + +RlsLb::RlsLb(Args args) + : LoadBalancingPolicy(std::move(args)), + instance_uuid_( + channel_args().GetOwnedString(GRPC_ARG_TEST_ONLY_RLS_INSTANCE_ID) + .value_or(GenerateUUID())), + cache_(this), + registered_metric_callback_( + channel_control_helper()->GetStatsPluginGroup().RegisterCallback( + [this](CallbackMetricReporter& reporter) { + MutexLock lock(&mu_); + cache_.ReportMetricsLocked(reporter); + }, + {kMetricCacheSize, kMetricCacheEntries})) { if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) { gpr_log(GPR_INFO, "[rlslb %p] policy created", this); } @@ -2025,6 +2136,7 @@ void RlsLb::ShutdownLocked() { if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_rls_trace)) { gpr_log(GPR_INFO, "[rlslb %p] policy shutdown", this); } + registered_metric_callback_.reset(); MutexLock lock(&mu_); is_shutdown_ = true; config_.reset(DEBUG_LOCATION, "ShutdownLocked"); @@ -2114,6 +2226,32 @@ void RlsLb::UpdatePickerLocked() { MakeRefCounted(RefAsSubclass(DEBUG_LOCATION, "Picker"))); } +void RlsLb::MaybeExportPickCount( + GlobalInstrumentsRegistry::GlobalUInt64CounterHandle handle, + absl::string_view target, const PickResult& pick_result) { + absl::string_view pick_result_string = + Match(pick_result.result, + [](const LoadBalancingPolicy::PickResult::Complete&) { + return "complete"; + }, + [](const LoadBalancingPolicy::PickResult::Queue&) { + return ""; + }, + [](const LoadBalancingPolicy::PickResult::Fail&) { + return "fail"; + }, + [](const LoadBalancingPolicy::PickResult::Drop&) { + return "drop"; + }); + if (pick_result_string.empty()) return; // Don't report queued picks. + auto& stats_plugins = channel_control_helper()->GetStatsPluginGroup(); + stats_plugins.AddCounter( + handle, 1, + {channel_control_helper()->GetTarget(), config_->lookup_service(), + target, pick_result_string}, + {}); +} + // // RlsLbFactory // diff --git a/src/core/load_balancing/rls/rls.h b/src/core/load_balancing/rls/rls.h new file mode 100644 index 00000000000..2be787b62d7 --- /dev/null +++ b/src/core/load_balancing/rls/rls.h @@ -0,0 +1,26 @@ +// +// Copyright 2024 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#ifndef GRPC_SRC_CORE_LOAD_BALANCING_RLS_RLS_H +#define GRPC_SRC_CORE_LOAD_BALANCING_RLS_RLS_H + +#include + +// A test-only channel arg to set the instance ID of the RLS LB +// policy for use in metric labels. +#define GRPC_ARG_TEST_ONLY_RLS_INSTANCE_ID "grpc.test-only.rls.instance_id" + +#endif // GRPC_SRC_CORE_LOAD_BALANCING_RLS_RLS_H diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 57a24b8d1fb..857b62a8259 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -558,6 +558,7 @@ CORE_SOURCE_FILES = [ 'src/core/lib/gprpp/time.cc', 'src/core/lib/gprpp/time_averaged_stats.cc', 'src/core/lib/gprpp/time_util.cc', + 'src/core/lib/gprpp/uuid_v4.cc', 'src/core/lib/gprpp/validation_errors.cc', 'src/core/lib/gprpp/windows/directory_reader.cc', 'src/core/lib/gprpp/windows/env.cc', diff --git a/test/core/channel/metrics_test.cc b/test/core/channel/metrics_test.cc index e27a8d09350..6cf9d47312c 100644 --- a/test/core/channel/metrics_test.cc +++ b/test/core/channel/metrics_test.cc @@ -197,6 +197,492 @@ TEST_F(MetricsTest, DoubleHistogram) { ::testing::Optional(::testing::UnorderedElementsAre(1.23, 2.34, 3.45))); } +TEST_F(MetricsTest, Int64Gauge) { + const absl::string_view kLabelKeys[] = {"label_key_1", "label_key_2"}; + const absl::string_view kOptionalLabelKeys[] = {"optional_label_key_1", + "optional_label_key_2"}; + auto int64_gauge_handle = GlobalInstrumentsRegistry::RegisterInt64Gauge( + "int64_gauge", "A simple int64 gauge.", "unit", kLabelKeys, + kOptionalLabelKeys, true); + constexpr absl::string_view kLabelValues[] = {"label_value_1", + "label_value_2"}; + constexpr absl::string_view kOptionalLabelValues[] = { + "optional_label_value_1", "optional_label_value_2"}; + constexpr absl::string_view kDomain1To4 = "domain1.domain2.domain3.domain4"; + constexpr absl::string_view kDomain2To4 = "domain2.domain3.domain4"; + constexpr absl::string_view kDomain3To4 = "domain3.domain4"; + auto plugin1 = MakeStatsPluginForTarget(kDomain1To4); + auto plugin2 = MakeStatsPluginForTarget(kDomain2To4); + auto plugin3 = MakeStatsPluginForTarget(kDomain3To4); + GlobalStatsPluginRegistry::GetStatsPluginsForChannel( + StatsPlugin::ChannelScope(kDomain1To4, "")) + .SetGauge(int64_gauge_handle, 1, kLabelValues, kOptionalLabelValues); + GlobalStatsPluginRegistry::GetStatsPluginsForChannel( + StatsPlugin::ChannelScope(kDomain2To4, "")) + .SetGauge(int64_gauge_handle, 2, kLabelValues, kOptionalLabelValues); + GlobalStatsPluginRegistry::GetStatsPluginsForChannel( + StatsPlugin::ChannelScope(kDomain3To4, "")) + .SetGauge(int64_gauge_handle, 3, kLabelValues, kOptionalLabelValues); + EXPECT_THAT(plugin1->GetGaugeValue(int64_gauge_handle, kLabelValues, + kOptionalLabelValues), + ::testing::Optional(1)); + EXPECT_THAT(plugin2->GetGaugeValue(int64_gauge_handle, kLabelValues, + kOptionalLabelValues), + ::testing::Optional(2)); + EXPECT_THAT(plugin3->GetGaugeValue(int64_gauge_handle, kLabelValues, + kOptionalLabelValues), + ::testing::Optional(3)); +} + +TEST_F(MetricsTest, DoubleGauge) { + const absl::string_view kLabelKeys[] = {"label_key_1", "label_key_2"}; + const absl::string_view kOptionalLabelKeys[] = {"optional_label_key_1", + "optional_label_key_2"}; + auto double_gauge_handle = GlobalInstrumentsRegistry::RegisterDoubleGauge( + "double_gauge", "A simple double gauge.", "unit", kLabelKeys, + kOptionalLabelKeys, true); + constexpr absl::string_view kLabelValues[] = {"label_value_1", + "label_value_2"}; + constexpr absl::string_view kOptionalLabelValues[] = { + "optional_label_value_1", "optional_label_value_2"}; + constexpr absl::string_view kDomain1To4 = "domain1.domain2.domain3.domain4"; + constexpr absl::string_view kDomain2To4 = "domain2.domain3.domain4"; + constexpr absl::string_view kDomain3To4 = "domain3.domain4"; + auto plugin1 = MakeStatsPluginForTarget(kDomain1To4); + auto plugin2 = MakeStatsPluginForTarget(kDomain2To4); + auto plugin3 = MakeStatsPluginForTarget(kDomain3To4); + GlobalStatsPluginRegistry::GetStatsPluginsForChannel( + StatsPlugin::ChannelScope(kDomain1To4, "")) + .SetGauge(double_gauge_handle, 1.23, kLabelValues, kOptionalLabelValues); + GlobalStatsPluginRegistry::GetStatsPluginsForChannel( + StatsPlugin::ChannelScope(kDomain2To4, "")) + .SetGauge(double_gauge_handle, 2.34, kLabelValues, kOptionalLabelValues); + GlobalStatsPluginRegistry::GetStatsPluginsForChannel( + StatsPlugin::ChannelScope(kDomain3To4, "")) + .SetGauge(double_gauge_handle, 3.45, kLabelValues, kOptionalLabelValues); + EXPECT_THAT(plugin1->GetGaugeValue(double_gauge_handle, kLabelValues, + kOptionalLabelValues), + ::testing::Optional(1.23)); + EXPECT_THAT(plugin2->GetGaugeValue(double_gauge_handle, kLabelValues, + kOptionalLabelValues), + ::testing::Optional(2.34)); + EXPECT_THAT(plugin3->GetGaugeValue(double_gauge_handle, kLabelValues, + kOptionalLabelValues), + ::testing::Optional(3.45)); +} + +TEST_F(MetricsTest, Int64CallbackGauge) { + const absl::string_view kLabelKeys[] = {"label_key_1", "label_key_2"}; + const absl::string_view kOptionalLabelKeys[] = {"optional_label_key_1", + "optional_label_key_2"}; + auto int64_gauge_handle = + GlobalInstrumentsRegistry::RegisterCallbackInt64Gauge( + "int64_gauge", "A simple int64 gauge.", "unit", kLabelKeys, + kOptionalLabelKeys, true); + constexpr absl::string_view kLabelValues[] = {"label_value_1", + "label_value_2"}; + constexpr absl::string_view kLabelValues2[] = {"label_value_3", + "label_value_2"}; + constexpr absl::string_view kOptionalLabelValues[] = { + "optional_label_value_1", "optional_label_value_2"}; + constexpr absl::string_view kDomain1To4 = "domain1.domain2.domain3.domain4"; + constexpr absl::string_view kDomain2To4 = "domain2.domain3.domain4"; + constexpr absl::string_view kDomain3To4 = "domain3.domain4"; + auto plugin1 = MakeStatsPluginForTarget(kDomain3To4); + auto plugin2 = MakeStatsPluginForTarget(kDomain2To4); + auto plugin3 = MakeStatsPluginForTarget(kDomain1To4); + // Register two callbacks that set the same metric but with different + // label values. The callbacks get used only by plugin1. + gpr_log(GPR_INFO, "testing callbacks for: plugin1"); + auto group1 = GlobalStatsPluginRegistry::GetStatsPluginsForChannel( + StatsPlugin::ChannelScope(kDomain3To4, "")); + auto callback1 = group1.RegisterCallback( + [&](CallbackMetricReporter& reporter) { + reporter.Report(int64_gauge_handle, 1, kLabelValues, + kOptionalLabelValues); + }, + {int64_gauge_handle}); + auto callback2 = group1.RegisterCallback( + [&](CallbackMetricReporter& reporter) { + reporter.Report(int64_gauge_handle, 2, kLabelValues2, + kOptionalLabelValues); + }, + {int64_gauge_handle}); + // No plugins have data yet. + EXPECT_EQ(plugin1->GetCallbackGaugeValue(int64_gauge_handle, kLabelValues, + kOptionalLabelValues), + absl::nullopt); + EXPECT_EQ(plugin1->GetCallbackGaugeValue(int64_gauge_handle, kLabelValues2, + kOptionalLabelValues), + absl::nullopt); + EXPECT_EQ(plugin2->GetCallbackGaugeValue(int64_gauge_handle, kLabelValues, + kOptionalLabelValues), + absl::nullopt); + EXPECT_EQ(plugin2->GetCallbackGaugeValue(int64_gauge_handle, kLabelValues2, + kOptionalLabelValues), + absl::nullopt); + EXPECT_EQ(plugin3->GetCallbackGaugeValue(int64_gauge_handle, kLabelValues, + kOptionalLabelValues), + absl::nullopt); + EXPECT_EQ(plugin3->GetCallbackGaugeValue(int64_gauge_handle, kLabelValues2, + kOptionalLabelValues), + absl::nullopt); + // Now invoke the callbacks. + plugin1->TriggerCallbacks(); + plugin2->TriggerCallbacks(); + plugin3->TriggerCallbacks(); + // Now plugin1 should have data, but the others should not. + EXPECT_THAT(plugin1->GetCallbackGaugeValue(int64_gauge_handle, kLabelValues, + kOptionalLabelValues), + ::testing::Optional(1)); + EXPECT_THAT(plugin1->GetCallbackGaugeValue(int64_gauge_handle, kLabelValues2, + kOptionalLabelValues), + ::testing::Optional(2)); + EXPECT_EQ(plugin2->GetCallbackGaugeValue(int64_gauge_handle, kLabelValues, + kOptionalLabelValues), + absl::nullopt); + EXPECT_EQ(plugin2->GetCallbackGaugeValue(int64_gauge_handle, kLabelValues2, + kOptionalLabelValues), + absl::nullopt); + EXPECT_EQ(plugin3->GetCallbackGaugeValue(int64_gauge_handle, kLabelValues, + kOptionalLabelValues), + absl::nullopt); + EXPECT_EQ(plugin3->GetCallbackGaugeValue(int64_gauge_handle, kLabelValues2, + kOptionalLabelValues), + absl::nullopt); + // De-register the callbacks. + callback1.reset(); + callback2.reset(); + // Now register callbacks that hit both plugin1 and plugin2. + gpr_log(GPR_INFO, "testing callbacks for: plugin1, plugin2"); + auto group2 = GlobalStatsPluginRegistry::GetStatsPluginsForChannel( + StatsPlugin::ChannelScope(kDomain2To4, "")); + callback1 = group2.RegisterCallback( + [&](CallbackMetricReporter& reporter) { + reporter.Report(int64_gauge_handle, 3, kLabelValues, + kOptionalLabelValues); + }, + {int64_gauge_handle}); + callback2 = group2.RegisterCallback( + [&](CallbackMetricReporter& reporter) { + reporter.Report(int64_gauge_handle, 4, kLabelValues2, + kOptionalLabelValues); + }, + {int64_gauge_handle}); + // Plugin1 still has data from before, but the others have none. + EXPECT_THAT(plugin1->GetCallbackGaugeValue(int64_gauge_handle, kLabelValues, + kOptionalLabelValues), + ::testing::Optional(1)); + EXPECT_THAT(plugin1->GetCallbackGaugeValue(int64_gauge_handle, kLabelValues2, + kOptionalLabelValues), + ::testing::Optional(2)); + EXPECT_EQ(plugin2->GetCallbackGaugeValue(int64_gauge_handle, kLabelValues, + kOptionalLabelValues), + absl::nullopt); + EXPECT_EQ(plugin2->GetCallbackGaugeValue(int64_gauge_handle, kLabelValues2, + kOptionalLabelValues), + absl::nullopt); + EXPECT_EQ(plugin3->GetCallbackGaugeValue(int64_gauge_handle, kLabelValues, + kOptionalLabelValues), + absl::nullopt); + EXPECT_EQ(plugin3->GetCallbackGaugeValue(int64_gauge_handle, kLabelValues2, + kOptionalLabelValues), + absl::nullopt); + // Now invoke the callbacks. + plugin1->TriggerCallbacks(); + plugin2->TriggerCallbacks(); + plugin3->TriggerCallbacks(); + // Now plugin1 and plugin2 should have data, but plugin3 should not. + EXPECT_THAT(plugin1->GetCallbackGaugeValue(int64_gauge_handle, kLabelValues, + kOptionalLabelValues), + ::testing::Optional(3)); + EXPECT_THAT(plugin1->GetCallbackGaugeValue(int64_gauge_handle, kLabelValues2, + kOptionalLabelValues), + ::testing::Optional(4)); + EXPECT_THAT(plugin2->GetCallbackGaugeValue(int64_gauge_handle, kLabelValues, + kOptionalLabelValues), + ::testing::Optional(3)); + EXPECT_THAT(plugin2->GetCallbackGaugeValue(int64_gauge_handle, kLabelValues2, + kOptionalLabelValues), + ::testing::Optional(4)); + EXPECT_EQ(plugin3->GetCallbackGaugeValue(int64_gauge_handle, kLabelValues, + kOptionalLabelValues), + absl::nullopt); + EXPECT_EQ(plugin3->GetCallbackGaugeValue(int64_gauge_handle, kLabelValues2, + kOptionalLabelValues), + absl::nullopt); + // De-register the callbacks. + callback1.reset(); + callback2.reset(); + // Now register callbacks that hit all three plugins. + gpr_log(GPR_INFO, "testing callbacks for: plugin1, plugin2, plugin3"); + auto group3 = GlobalStatsPluginRegistry::GetStatsPluginsForChannel( + StatsPlugin::ChannelScope(kDomain1To4, "")); + callback1 = group3.RegisterCallback( + [&](CallbackMetricReporter& reporter) { + reporter.Report(int64_gauge_handle, 5, kLabelValues, + kOptionalLabelValues); + }, + {int64_gauge_handle}); + callback2 = group3.RegisterCallback( + [&](CallbackMetricReporter& reporter) { + reporter.Report(int64_gauge_handle, 6, kLabelValues2, + kOptionalLabelValues); + }, + {int64_gauge_handle}); + // Plugin1 and plugin2 still has data from before, but plugin3 has none. + EXPECT_THAT(plugin1->GetCallbackGaugeValue(int64_gauge_handle, kLabelValues, + kOptionalLabelValues), + ::testing::Optional(3)); + EXPECT_THAT(plugin1->GetCallbackGaugeValue(int64_gauge_handle, kLabelValues2, + kOptionalLabelValues), + ::testing::Optional(4)); + EXPECT_THAT(plugin2->GetCallbackGaugeValue(int64_gauge_handle, kLabelValues, + kOptionalLabelValues), + ::testing::Optional(3)); + EXPECT_THAT(plugin2->GetCallbackGaugeValue(int64_gauge_handle, kLabelValues2, + kOptionalLabelValues), + ::testing::Optional(4)); + EXPECT_EQ(plugin3->GetCallbackGaugeValue(int64_gauge_handle, kLabelValues, + kOptionalLabelValues), + absl::nullopt); + EXPECT_EQ(plugin3->GetCallbackGaugeValue(int64_gauge_handle, kLabelValues2, + kOptionalLabelValues), + absl::nullopt); + // Now invoke the callbacks. + plugin1->TriggerCallbacks(); + plugin2->TriggerCallbacks(); + plugin3->TriggerCallbacks(); + // Now plugin1 and plugin2 should have data, but plugin3 should not. + EXPECT_THAT(plugin1->GetCallbackGaugeValue(int64_gauge_handle, kLabelValues, + kOptionalLabelValues), + ::testing::Optional(5)); + EXPECT_THAT(plugin1->GetCallbackGaugeValue(int64_gauge_handle, kLabelValues2, + kOptionalLabelValues), + ::testing::Optional(6)); + EXPECT_THAT(plugin2->GetCallbackGaugeValue(int64_gauge_handle, kLabelValues, + kOptionalLabelValues), + ::testing::Optional(5)); + EXPECT_THAT(plugin2->GetCallbackGaugeValue(int64_gauge_handle, kLabelValues2, + kOptionalLabelValues), + ::testing::Optional(6)); + EXPECT_THAT(plugin3->GetCallbackGaugeValue(int64_gauge_handle, kLabelValues, + kOptionalLabelValues), + ::testing::Optional(5)); + EXPECT_THAT(plugin3->GetCallbackGaugeValue(int64_gauge_handle, kLabelValues2, + kOptionalLabelValues), + ::testing::Optional(6)); + // Need to destroy callbacks before the plugin group that created them. + callback1.reset(); + callback2.reset(); +} + +TEST_F(MetricsTest, DoubleCallbackGauge) { + const absl::string_view kLabelKeys[] = {"label_key_1", "label_key_2"}; + const absl::string_view kOptionalLabelKeys[] = {"optional_label_key_1", + "optional_label_key_2"}; + auto double_gauge_handle = + GlobalInstrumentsRegistry::RegisterCallbackDoubleGauge( + "double_gauge", "A simple double gauge.", "unit", kLabelKeys, + kOptionalLabelKeys, true); + constexpr absl::string_view kLabelValues[] = {"label_value_1", + "label_value_2"}; + constexpr absl::string_view kLabelValues2[] = {"label_value_3", + "label_value_2"}; + constexpr absl::string_view kOptionalLabelValues[] = { + "optional_label_value_1", "optional_label_value_2"}; + constexpr absl::string_view kDomain1To4 = "domain1.domain2.domain3.domain4"; + constexpr absl::string_view kDomain2To4 = "domain2.domain3.domain4"; + constexpr absl::string_view kDomain3To4 = "domain3.domain4"; + auto plugin1 = MakeStatsPluginForTarget(kDomain3To4); + auto plugin2 = MakeStatsPluginForTarget(kDomain2To4); + auto plugin3 = MakeStatsPluginForTarget(kDomain1To4); + // Register two callbacks that set the same metric but with different + // label values. The callbacks get used only by plugin1. + gpr_log(GPR_INFO, "testing callbacks for: plugin1"); + auto group1 = GlobalStatsPluginRegistry::GetStatsPluginsForChannel( + StatsPlugin::ChannelScope(kDomain3To4, "")); + auto callback1 = group1.RegisterCallback( + [&](CallbackMetricReporter& reporter) { + reporter.Report(double_gauge_handle, 1.23, kLabelValues, + kOptionalLabelValues); + }, + {double_gauge_handle}); + auto callback2 = group1.RegisterCallback( + [&](CallbackMetricReporter& reporter) { + reporter.Report(double_gauge_handle, 2.34, kLabelValues2, + kOptionalLabelValues); + }, + {double_gauge_handle}); + // No plugins have data yet. + EXPECT_EQ(plugin1->GetCallbackGaugeValue(double_gauge_handle, kLabelValues, + kOptionalLabelValues), + absl::nullopt); + EXPECT_EQ(plugin1->GetCallbackGaugeValue(double_gauge_handle, kLabelValues2, + kOptionalLabelValues), + absl::nullopt); + EXPECT_EQ(plugin2->GetCallbackGaugeValue(double_gauge_handle, kLabelValues, + kOptionalLabelValues), + absl::nullopt); + EXPECT_EQ(plugin2->GetCallbackGaugeValue(double_gauge_handle, kLabelValues2, + kOptionalLabelValues), + absl::nullopt); + EXPECT_EQ(plugin3->GetCallbackGaugeValue(double_gauge_handle, kLabelValues, + kOptionalLabelValues), + absl::nullopt); + EXPECT_EQ(plugin3->GetCallbackGaugeValue(double_gauge_handle, kLabelValues2, + kOptionalLabelValues), + absl::nullopt); + // Now invoke the callbacks. + plugin1->TriggerCallbacks(); + plugin2->TriggerCallbacks(); + plugin3->TriggerCallbacks(); + // Now plugin1 should have data, but the others should not. + EXPECT_THAT(plugin1->GetCallbackGaugeValue(double_gauge_handle, kLabelValues, + kOptionalLabelValues), + ::testing::Optional(1.23)); + EXPECT_THAT(plugin1->GetCallbackGaugeValue(double_gauge_handle, kLabelValues2, + kOptionalLabelValues), + ::testing::Optional(2.34)); + EXPECT_EQ(plugin2->GetCallbackGaugeValue(double_gauge_handle, kLabelValues, + kOptionalLabelValues), + absl::nullopt); + EXPECT_EQ(plugin2->GetCallbackGaugeValue(double_gauge_handle, kLabelValues2, + kOptionalLabelValues), + absl::nullopt); + EXPECT_EQ(plugin3->GetCallbackGaugeValue(double_gauge_handle, kLabelValues, + kOptionalLabelValues), + absl::nullopt); + EXPECT_EQ(plugin3->GetCallbackGaugeValue(double_gauge_handle, kLabelValues2, + kOptionalLabelValues), + absl::nullopt); + // De-register the callbacks. + callback1.reset(); + callback2.reset(); + // Now register callbacks that hit both plugin1 and plugin2. + gpr_log(GPR_INFO, "testing callbacks for: plugin1, plugin2"); + auto group2 = GlobalStatsPluginRegistry::GetStatsPluginsForChannel( + StatsPlugin::ChannelScope(kDomain2To4, "")); + callback1 = group2.RegisterCallback( + [&](CallbackMetricReporter& reporter) { + reporter.Report(double_gauge_handle, 3.45, kLabelValues, + kOptionalLabelValues); + }, + {double_gauge_handle}); + callback2 = group2.RegisterCallback( + [&](CallbackMetricReporter& reporter) { + reporter.Report(double_gauge_handle, 4.56, kLabelValues2, + kOptionalLabelValues); + }, + {double_gauge_handle}); + // Plugin1 still has data from before, but the others have none. + EXPECT_THAT(plugin1->GetCallbackGaugeValue(double_gauge_handle, kLabelValues, + kOptionalLabelValues), + ::testing::Optional(1.23)); + EXPECT_THAT(plugin1->GetCallbackGaugeValue(double_gauge_handle, kLabelValues2, + kOptionalLabelValues), + ::testing::Optional(2.34)); + EXPECT_EQ(plugin2->GetCallbackGaugeValue(double_gauge_handle, kLabelValues, + kOptionalLabelValues), + absl::nullopt); + EXPECT_EQ(plugin2->GetCallbackGaugeValue(double_gauge_handle, kLabelValues2, + kOptionalLabelValues), + absl::nullopt); + EXPECT_EQ(plugin3->GetCallbackGaugeValue(double_gauge_handle, kLabelValues, + kOptionalLabelValues), + absl::nullopt); + EXPECT_EQ(plugin3->GetCallbackGaugeValue(double_gauge_handle, kLabelValues2, + kOptionalLabelValues), + absl::nullopt); + // Now invoke the callbacks. + plugin1->TriggerCallbacks(); + plugin2->TriggerCallbacks(); + plugin3->TriggerCallbacks(); + // Now plugin1 and plugin2 should have data, but plugin3 should not. + EXPECT_THAT(plugin1->GetCallbackGaugeValue(double_gauge_handle, kLabelValues, + kOptionalLabelValues), + ::testing::Optional(3.45)); + EXPECT_THAT(plugin1->GetCallbackGaugeValue(double_gauge_handle, kLabelValues2, + kOptionalLabelValues), + ::testing::Optional(4.56)); + EXPECT_THAT(plugin2->GetCallbackGaugeValue(double_gauge_handle, kLabelValues, + kOptionalLabelValues), + ::testing::Optional(3.45)); + EXPECT_THAT(plugin2->GetCallbackGaugeValue(double_gauge_handle, kLabelValues2, + kOptionalLabelValues), + ::testing::Optional(4.56)); + EXPECT_EQ(plugin3->GetCallbackGaugeValue(double_gauge_handle, kLabelValues, + kOptionalLabelValues), + absl::nullopt); + EXPECT_EQ(plugin3->GetCallbackGaugeValue(double_gauge_handle, kLabelValues2, + kOptionalLabelValues), + absl::nullopt); + // De-register the callbacks. + callback1.reset(); + callback2.reset(); + // Now register callbacks that hit all three plugins. + gpr_log(GPR_INFO, "testing callbacks for: plugin1, plugin2, plugin3"); + auto group3 = GlobalStatsPluginRegistry::GetStatsPluginsForChannel( + StatsPlugin::ChannelScope(kDomain1To4, "")); + callback1 = group3.RegisterCallback( + [&](CallbackMetricReporter& reporter) { + reporter.Report(double_gauge_handle, 5.67, kLabelValues, + kOptionalLabelValues); + }, + {double_gauge_handle}); + callback2 = group3.RegisterCallback( + [&](CallbackMetricReporter& reporter) { + reporter.Report(double_gauge_handle, 6.78, kLabelValues2, + kOptionalLabelValues); + }, + {double_gauge_handle}); + // Plugin1 and plugin2 still has data from before, but plugin3 has none. + EXPECT_THAT(plugin1->GetCallbackGaugeValue(double_gauge_handle, kLabelValues, + kOptionalLabelValues), + ::testing::Optional(3.45)); + EXPECT_THAT(plugin1->GetCallbackGaugeValue(double_gauge_handle, kLabelValues2, + kOptionalLabelValues), + ::testing::Optional(4.56)); + EXPECT_THAT(plugin2->GetCallbackGaugeValue(double_gauge_handle, kLabelValues, + kOptionalLabelValues), + ::testing::Optional(3.45)); + EXPECT_THAT(plugin2->GetCallbackGaugeValue(double_gauge_handle, kLabelValues2, + kOptionalLabelValues), + ::testing::Optional(4.56)); + EXPECT_EQ(plugin3->GetCallbackGaugeValue(double_gauge_handle, kLabelValues, + kOptionalLabelValues), + absl::nullopt); + EXPECT_EQ(plugin3->GetCallbackGaugeValue(double_gauge_handle, kLabelValues2, + kOptionalLabelValues), + absl::nullopt); + // Now invoke the callbacks. + plugin1->TriggerCallbacks(); + plugin2->TriggerCallbacks(); + plugin3->TriggerCallbacks(); + // Now plugin1 and plugin2 should have data, but plugin3 should not. + EXPECT_THAT(plugin1->GetCallbackGaugeValue(double_gauge_handle, kLabelValues, + kOptionalLabelValues), + ::testing::Optional(5.67)); + EXPECT_THAT(plugin1->GetCallbackGaugeValue(double_gauge_handle, kLabelValues2, + kOptionalLabelValues), + ::testing::Optional(6.78)); + EXPECT_THAT(plugin2->GetCallbackGaugeValue(double_gauge_handle, kLabelValues, + kOptionalLabelValues), + ::testing::Optional(5.67)); + EXPECT_THAT(plugin2->GetCallbackGaugeValue(double_gauge_handle, kLabelValues2, + kOptionalLabelValues), + ::testing::Optional(6.78)); + EXPECT_THAT(plugin3->GetCallbackGaugeValue(double_gauge_handle, kLabelValues, + kOptionalLabelValues), + ::testing::Optional(5.67)); + EXPECT_THAT(plugin3->GetCallbackGaugeValue(double_gauge_handle, kLabelValues2, + kOptionalLabelValues), + ::testing::Optional(6.78)); + // Need to destroy callbacks before the plugin group that created them. + callback1.reset(); + callback2.reset(); +} + TEST_F(MetricsTest, DisableByDefaultMetricIsNotRecordedByFakeStatsPlugin) { const absl::string_view kLabelKeys[] = {"label_key_1", "label_key_2"}; const absl::string_view kOptionalLabelKeys[] = {"optional_label_key_1", diff --git a/test/core/util/fake_stats_plugin.cc b/test/core/util/fake_stats_plugin.cc index 9aa31783e0e..d00df284599 100644 --- a/test/core/util/fake_stats_plugin.cc +++ b/test/core/util/fake_stats_plugin.cc @@ -175,6 +175,44 @@ GlobalInstrumentsRegistryTestPeer::FindDoubleHistogramHandleByName( GlobalInstrumentsRegistry::InstrumentType::kHistogram); } +absl::optional +GlobalInstrumentsRegistryTestPeer::FindInt64GaugeHandleByName( + absl::string_view name) { + return FindInstrument( + GlobalInstrumentsRegistry::GetInstrumentList(), name, + GlobalInstrumentsRegistry::ValueType::kInt64, + GlobalInstrumentsRegistry::InstrumentType::kGauge); +} + +absl::optional +GlobalInstrumentsRegistryTestPeer::FindDoubleGaugeHandleByName( + absl::string_view name) { + return FindInstrument( + GlobalInstrumentsRegistry::GetInstrumentList(), name, + GlobalInstrumentsRegistry::ValueType::kDouble, + GlobalInstrumentsRegistry::InstrumentType::kGauge); +} + +absl::optional +GlobalInstrumentsRegistryTestPeer::FindCallbackInt64GaugeHandleByName( + absl::string_view name) { + return FindInstrument< + GlobalInstrumentsRegistry::GlobalCallbackInt64GaugeHandle>( + GlobalInstrumentsRegistry::GetInstrumentList(), name, + GlobalInstrumentsRegistry::ValueType::kInt64, + GlobalInstrumentsRegistry::InstrumentType::kCallbackGauge); +} + +absl::optional +GlobalInstrumentsRegistryTestPeer::FindCallbackDoubleGaugeHandleByName( + absl::string_view name) { + return FindInstrument< + GlobalInstrumentsRegistry::GlobalCallbackDoubleGaugeHandle>( + GlobalInstrumentsRegistry::GetInstrumentList(), name, + GlobalInstrumentsRegistry::ValueType::kDouble, + GlobalInstrumentsRegistry::InstrumentType::kCallbackGauge); +} + GlobalInstrumentsRegistry::GlobalInstrumentDescriptor* GlobalInstrumentsRegistryTestPeer::FindMetricDescriptorByName( absl::string_view name) { diff --git a/test/core/util/fake_stats_plugin.h b/test/core/util/fake_stats_plugin.h index a7a4751ebbd..04268ca694e 100644 --- a/test/core/util/fake_stats_plugin.h +++ b/test/core/util/fake_stats_plugin.h @@ -219,23 +219,41 @@ class FakeStatsPlugin : public StatsPlugin { std::string(descriptor.name).c_str()); return; } - if (descriptor.instrument_type == - GlobalInstrumentsRegistry::InstrumentType::kCounter) { - if (descriptor.value_type == - GlobalInstrumentsRegistry::ValueType::kUInt64) { - uint64_counters_.emplace(descriptor.index, descriptor); - } else { - double_counters_.emplace(descriptor.index, descriptor); - } - } else { - EXPECT_EQ(descriptor.instrument_type, - GlobalInstrumentsRegistry::InstrumentType::kHistogram); - if (descriptor.value_type == - GlobalInstrumentsRegistry::ValueType::kUInt64) { - uint64_histograms_.emplace(descriptor.index, descriptor); - } else { - double_histograms_.emplace(descriptor.index, descriptor); - } + switch (descriptor.instrument_type) { + case GlobalInstrumentsRegistry::InstrumentType::kCounter: + if (descriptor.value_type == + GlobalInstrumentsRegistry::ValueType::kUInt64) { + uint64_counters_.emplace(descriptor.index, descriptor); + } else { + double_counters_.emplace(descriptor.index, descriptor); + } + break; + case GlobalInstrumentsRegistry::InstrumentType::kHistogram: + if (descriptor.value_type == + GlobalInstrumentsRegistry::ValueType::kUInt64) { + uint64_histograms_.emplace(descriptor.index, descriptor); + } else { + double_histograms_.emplace(descriptor.index, descriptor); + } + break; + case GlobalInstrumentsRegistry::InstrumentType::kGauge: + if (descriptor.value_type == + GlobalInstrumentsRegistry::ValueType::kInt64) { + int64_gauges_.emplace(descriptor.index, descriptor); + } else { + double_gauges_.emplace(descriptor.index, descriptor); + } + break; + case GlobalInstrumentsRegistry::InstrumentType::kCallbackGauge: + if (descriptor.value_type == + GlobalInstrumentsRegistry::ValueType::kInt64) { + int64_callback_gauges_.emplace(descriptor.index, descriptor); + } else { + double_callback_gauges_.emplace(descriptor.index, descriptor); + } + break; + default: + Crash("unknown instrument type"); } }); } @@ -314,6 +332,41 @@ class FakeStatsPlugin : public StatsPlugin { if (iter == double_histograms_.end()) return; iter->second.Record(value, label_values, optional_values); } + void SetGauge(GlobalInstrumentsRegistry::GlobalInt64GaugeHandle handle, + int64_t value, absl::Span label_values, + absl::Span optional_values) override { + gpr_log(GPR_INFO, + "FakeStatsPlugin[%p]::RecordGauge(index=%u, value=(uint64)%lu, " + "label_values={%s}, optional_label_values={%s}", + this, handle.index, value, + absl::StrJoin(label_values, ", ").c_str(), + absl::StrJoin(optional_values, ", ").c_str()); + auto iter = int64_gauges_.find(handle.index); + if (iter == int64_gauges_.end()) return; + iter->second.Set(value, label_values, optional_values); + } + void SetGauge(GlobalInstrumentsRegistry::GlobalDoubleGaugeHandle handle, + double value, absl::Span label_values, + absl::Span optional_values) override { + gpr_log(GPR_INFO, + "FakeStatsPlugin[%p]::RecordGauge(index=%u, value=(double)%f, " + "label_values={%s}, optional_label_values={%s}", + this, handle.index, value, + absl::StrJoin(label_values, ", ").c_str(), + absl::StrJoin(optional_values, ", ").c_str()); + auto iter = double_gauges_.find(handle.index); + if (iter == double_gauges_.end()) return; + iter->second.Set(value, label_values, optional_values); + } + void AddCallback(RegisteredMetricCallback* callback) override { + gpr_log(GPR_INFO, "FakeStatsPlugin[%p]::AddCallback(%p)", this, callback); + callbacks_.insert(callback); + } + void RemoveCallback(RegisteredMetricCallback* callback) override { + gpr_log(GPR_INFO, "FakeStatsPlugin[%p]::RemoveCallback(%p)", this, + callback); + callbacks_.erase(callback); + } absl::optional GetCounterValue( GlobalInstrumentsRegistry::GlobalUInt64CounterHandle handle, @@ -355,8 +408,96 @@ class FakeStatsPlugin : public StatsPlugin { } return iter->second.GetValues(label_values, optional_values); } + absl::optional GetGaugeValue( + GlobalInstrumentsRegistry::GlobalInt64GaugeHandle handle, + absl::Span label_values, + absl::Span optional_values) { + auto iter = int64_gauges_.find(handle.index); + if (iter == int64_gauges_.end()) { + return absl::nullopt; + } + return iter->second.GetValue(label_values, optional_values); + } + absl::optional GetGaugeValue( + GlobalInstrumentsRegistry::GlobalDoubleGaugeHandle handle, + absl::Span label_values, + absl::Span optional_values) { + auto iter = double_gauges_.find(handle.index); + if (iter == double_gauges_.end()) { + return absl::nullopt; + } + return iter->second.GetValue(label_values, optional_values); + } + void TriggerCallbacks() { + gpr_log(GPR_INFO, "FakeStatsPlugin[%p]::TriggerCallbacks(): START", this); + Reporter reporter(*this); + for (auto* callback : callbacks_) { + callback->Run(reporter); + } + gpr_log(GPR_INFO, "FakeStatsPlugin[%p]::TriggerCallbacks(): END", this); + } + absl::optional GetCallbackGaugeValue( + GlobalInstrumentsRegistry::GlobalCallbackInt64GaugeHandle handle, + absl::Span label_values, + absl::Span optional_values) { + auto iter = int64_callback_gauges_.find(handle.index); + if (iter == int64_callback_gauges_.end()) { + return absl::nullopt; + } + return iter->second.GetValue(label_values, optional_values); + } + absl::optional GetCallbackGaugeValue( + GlobalInstrumentsRegistry::GlobalCallbackDoubleGaugeHandle handle, + absl::Span label_values, + absl::Span optional_values) { + auto iter = double_callback_gauges_.find(handle.index); + if (iter == double_callback_gauges_.end()) { + return absl::nullopt; + } + return iter->second.GetValue(label_values, optional_values); + } private: + class Reporter : public CallbackMetricReporter { + public: + explicit Reporter(FakeStatsPlugin& plugin) : plugin_(plugin) {} + + void Report( + GlobalInstrumentsRegistry::GlobalCallbackInt64GaugeHandle handle, + int64_t value, absl::Span label_values, + absl::Span optional_values) override { + gpr_log(GPR_INFO, + "FakeStatsPlugin[%p]::Reporter::Report(index=%u, " + "value=(uint64)%ld, label_values={%s}, " + "optional_label_values={%s}", + this, handle.index, value, + absl::StrJoin(label_values, ", ").c_str(), + absl::StrJoin(optional_values, ", ").c_str()); + auto iter = plugin_.int64_callback_gauges_.find(handle.index); + if (iter == plugin_.int64_callback_gauges_.end()) return; + iter->second.Set(value, label_values, optional_values); + } + + void Report( + GlobalInstrumentsRegistry::GlobalCallbackDoubleGaugeHandle handle, + double value, absl::Span label_values, + absl::Span optional_values) override { + gpr_log(GPR_INFO, + "FakeStatsPlugin[%p]::Reporter::Report(index=%u, " + "value=(double)%f, label_values={%s}, " + "optional_label_values={%s}", + this, handle.index, value, + absl::StrJoin(label_values, ", ").c_str(), + absl::StrJoin(optional_values, ", ").c_str()); + auto iter = plugin_.double_callback_gauges_.find(handle.index); + if (iter == plugin_.double_callback_gauges_.end()) return; + iter->second.Set(value, label_values, optional_values); + } + + private: + FakeStatsPlugin& plugin_; + }; + template class Counter { public: @@ -442,12 +583,53 @@ class FakeStatsPlugin : public StatsPlugin { absl::flat_hash_map> storage_; }; + template + class Gauge { + public: + explicit Gauge(GlobalInstrumentsRegistry::GlobalInstrumentDescriptor u) + : name_(u.name), + description_(u.description), + unit_(u.unit), + label_keys_(std::move(u.label_keys)), + optional_label_keys_(std::move(u.optional_label_keys)) {} + + void Set(T t, absl::Span label_values, + absl::Span optional_values) { + storage_[MakeLabelString(label_keys_, label_values, optional_label_keys_, + optional_values)] = t; + } + + absl::optional GetValue( + absl::Span label_values, + absl::Span optional_values) { + auto iter = storage_.find(MakeLabelString( + label_keys_, label_values, optional_label_keys_, optional_values)); + if (iter == storage_.end()) { + return absl::nullopt; + } + return iter->second; + } + + private: + absl::string_view name_; + absl::string_view description_; + absl::string_view unit_; + std::vector label_keys_; + std::vector optional_label_keys_; + absl::flat_hash_map storage_; + }; + absl::AnyInvocable channel_filter_; // Instruments. absl::flat_hash_map> uint64_counters_; absl::flat_hash_map> double_counters_; absl::flat_hash_map> uint64_histograms_; absl::flat_hash_map> double_histograms_; + absl::flat_hash_map> int64_gauges_; + absl::flat_hash_map> double_gauges_; + absl::flat_hash_map> int64_callback_gauges_; + absl::flat_hash_map> double_callback_gauges_; + std::set callbacks_; }; class FakeStatsPluginBuilder { @@ -492,6 +674,16 @@ class GlobalInstrumentsRegistryTestPeer { FindUInt64HistogramHandleByName(absl::string_view name); static absl::optional FindDoubleHistogramHandleByName(absl::string_view name); + static absl::optional + FindInt64GaugeHandleByName(absl::string_view name); + static absl::optional + FindDoubleGaugeHandleByName(absl::string_view name); + static absl::optional< + GlobalInstrumentsRegistry::GlobalCallbackInt64GaugeHandle> + FindCallbackInt64GaugeHandleByName(absl::string_view name); + static absl::optional< + GlobalInstrumentsRegistry::GlobalCallbackDoubleGaugeHandle> + FindCallbackDoubleGaugeHandleByName(absl::string_view name); static GlobalInstrumentsRegistry::GlobalInstrumentDescriptor* FindMetricDescriptorByName(absl::string_view name); diff --git a/test/cpp/end2end/BUILD b/test/cpp/end2end/BUILD index c06aafc882e..f0f5305a67d 100644 --- a/test/cpp/end2end/BUILD +++ b/test/cpp/end2end/BUILD @@ -568,6 +568,7 @@ grpc_cc_test( "//src/proto/grpc/testing:echo_messages_proto", "//src/proto/grpc/testing:echo_proto", "//src/proto/grpc/testing/duplicate:echo_duplicate_proto", + "//test/core/util:fake_stats_plugin", "//test/core/util:grpc_test_util", "//test/core/util:test_lb_policies", "//test/cpp/util:test_config", diff --git a/test/cpp/end2end/rls_end2end_test.cc b/test/cpp/end2end/rls_end2end_test.cc index faa798acfbc..ec48de736da 100644 --- a/test/cpp/end2end/rls_end2end_test.cc +++ b/test/cpp/end2end/rls_end2end_test.cc @@ -49,6 +49,7 @@ #include "src/core/lib/iomgr/sockaddr.h" #include "src/core/lib/security/credentials/fake/fake_credentials.h" #include "src/core/lib/uri/uri_parser.h" +#include "src/core/load_balancing/rls/rls.h" #include "src/core/resolver/fake/fake_resolver.h" #include "src/core/service_config/service_config_impl.h" #include "src/cpp/client/secure_credentials.h" @@ -56,6 +57,7 @@ #include "src/proto/grpc/lookup/v1/rls.grpc.pb.h" #include "src/proto/grpc/lookup/v1/rls.pb.h" #include "src/proto/grpc/testing/echo.grpc.pb.h" +#include "test/core/util/fake_stats_plugin.h" #include "test/core/util/port.h" #include "test/core/util/resolve_localhost_ip46.h" #include "test/core/util/test_config.h" @@ -73,6 +75,7 @@ namespace { const char* kServerName = "test.google.fr"; const char* kRequestMessage = "Live long and prosper."; +const char* kRlsInstanceUuid = "rls_instance_uuid"; const char* kCallCredsMdKey = "call_cred_name"; const char* kCallCredsMdValue = "call_cred_value"; @@ -182,6 +185,7 @@ class RlsEnd2endTest : public ::testing::Test { EXPECT_EQ(ctx->ExperimentalGetAuthority(), kServerName); }); rls_server_->Start(); + rls_server_target_ = absl::StrFormat("localhost:%d", rls_server_->port_); // Set up client. resolver_response_generator_ = std::make_unique(); @@ -189,6 +193,7 @@ class RlsEnd2endTest : public ::testing::Test { args.SetPointer(GRPC_ARG_FAKE_RESOLVER_RESPONSE_GENERATOR, resolver_response_generator_->Get()); args.SetString(GRPC_ARG_FAKE_SECURITY_EXPECTED_TARGETS, kServerName); + args.SetString(GRPC_ARG_TEST_ONLY_RLS_INSTANCE_ID, kRlsInstanceUuid); grpc_channel_credentials* channel_creds = grpc_fake_transport_security_credentials_create(); grpc_call_credentials* call_creds = grpc_md_only_test_credentials_create( @@ -198,8 +203,8 @@ class RlsEnd2endTest : public ::testing::Test { nullptr)); call_creds->Unref(); channel_creds->Unref(); - channel_ = grpc::CreateCustomChannel(absl::StrCat("fake:///", kServerName), - std::move(creds), args); + target_uri_ = absl::StrCat("fake:///", kServerName); + channel_ = grpc::CreateCustomChannel(target_uri_, std::move(creds), args); stub_ = grpc::testing::EchoTestService::NewStub(channel_); } @@ -295,8 +300,8 @@ class RlsEnd2endTest : public ::testing::Test { class ServiceConfigBuilder { public: - explicit ServiceConfigBuilder(int rls_server_port) - : rls_server_port_(rls_server_port) {} + explicit ServiceConfigBuilder(absl::string_view rls_server_target) + : rls_server_target_(rls_server_target) {} ServiceConfigBuilder& set_lookup_service_timeout( grpc_core::Duration timeout) { @@ -333,7 +338,7 @@ class RlsEnd2endTest : public ::testing::Test { // First build parts of routeLookupConfig. std::vector route_lookup_config_parts; route_lookup_config_parts.push_back(absl::StrFormat( - " \"lookupService\":\"localhost:%d\"", rls_server_port_)); + " \"lookupService\":\"%s\"", rls_server_target_)); if (lookup_service_timeout_ > grpc_core::Duration::Zero()) { route_lookup_config_parts.push_back( absl::StrFormat(" \"lookupServiceTimeout\":\"%fs\"", @@ -382,7 +387,7 @@ class RlsEnd2endTest : public ::testing::Test { } private: - int rls_server_port_; + absl::string_view rls_server_target_; grpc_core::Duration lookup_service_timeout_; std::string default_target_; grpc_core::Duration max_age_; @@ -392,7 +397,7 @@ class RlsEnd2endTest : public ::testing::Test { }; ServiceConfigBuilder MakeServiceConfigBuilder() { - return ServiceConfigBuilder(rls_server_->port_); + return ServiceConfigBuilder(rls_server_target_); } void SetNextResolution(absl::string_view service_config_json) { @@ -456,9 +461,11 @@ class RlsEnd2endTest : public ::testing::Test { }; std::vector>> backends_; + std::string rls_server_target_; std::unique_ptr> rls_server_; std::unique_ptr resolver_response_generator_; + std::string target_uri_; std::shared_ptr channel_; std::unique_ptr stub_; }; @@ -1370,6 +1377,309 @@ TEST_F(RlsEnd2endTest, ConnectivityStateTransientFailure) { channel_->GetState(/*try_to_connect=*/false)); } +class RlsMetricsEnd2endTest : public RlsEnd2endTest { + protected: + void SetUp() override { + // Register stats plugin before initializing client. + stats_plugin_ = grpc_core::FakeStatsPluginBuilder() + .UseDisabledByDefaultMetrics(true) + .BuildAndRegister(); + RlsEnd2endTest::SetUp(); + } + + std::shared_ptr stats_plugin_; +}; + +TEST_F(RlsMetricsEnd2endTest, MetricDefinitionDefaultTargetPicks) { + const auto* descriptor = + grpc_core::GlobalInstrumentsRegistryTestPeer::FindMetricDescriptorByName( + "grpc.lb.rls.default_target_picks"); + ASSERT_NE(descriptor, nullptr); + EXPECT_EQ(descriptor->value_type, + grpc_core::GlobalInstrumentsRegistry::ValueType::kUInt64); + EXPECT_EQ(descriptor->instrument_type, + grpc_core::GlobalInstrumentsRegistry::InstrumentType::kCounter); + EXPECT_EQ(descriptor->enable_by_default, false); + EXPECT_EQ(descriptor->name, "grpc.lb.rls.default_target_picks"); + EXPECT_EQ(descriptor->unit, "{pick}"); + EXPECT_THAT(descriptor->label_keys, + ::testing::ElementsAre("grpc.target", "grpc.lb.rls.server_target", + "grpc.lb.rls.data_plane_target", + "grpc.lb.pick_result")); + EXPECT_THAT(descriptor->optional_label_keys, ::testing::ElementsAre()); +} + +TEST_F(RlsMetricsEnd2endTest, MetricDefinitionTargetPicks) { + const auto* descriptor = + grpc_core::GlobalInstrumentsRegistryTestPeer::FindMetricDescriptorByName( + "grpc.lb.rls.target_picks"); + ASSERT_NE(descriptor, nullptr); + EXPECT_EQ(descriptor->value_type, + grpc_core::GlobalInstrumentsRegistry::ValueType::kUInt64); + EXPECT_EQ(descriptor->instrument_type, + grpc_core::GlobalInstrumentsRegistry::InstrumentType::kCounter); + EXPECT_EQ(descriptor->enable_by_default, false); + EXPECT_EQ(descriptor->name, "grpc.lb.rls.target_picks"); + EXPECT_EQ(descriptor->unit, "{pick}"); + EXPECT_THAT(descriptor->label_keys, + ::testing::ElementsAre("grpc.target", "grpc.lb.rls.server_target", + "grpc.lb.rls.data_plane_target", + "grpc.lb.pick_result")); + EXPECT_THAT(descriptor->optional_label_keys, ::testing::ElementsAre()); +} + +TEST_F(RlsMetricsEnd2endTest, MetricDefinitionFailedPicks) { + const auto* descriptor = + grpc_core::GlobalInstrumentsRegistryTestPeer::FindMetricDescriptorByName( + "grpc.lb.rls.failed_picks"); + ASSERT_NE(descriptor, nullptr); + EXPECT_EQ(descriptor->value_type, + grpc_core::GlobalInstrumentsRegistry::ValueType::kUInt64); + EXPECT_EQ(descriptor->instrument_type, + grpc_core::GlobalInstrumentsRegistry::InstrumentType::kCounter); + EXPECT_EQ(descriptor->enable_by_default, false); + EXPECT_EQ(descriptor->name, "grpc.lb.rls.failed_picks"); + EXPECT_EQ(descriptor->unit, "{pick}"); + EXPECT_THAT( + descriptor->label_keys, + ::testing::ElementsAre("grpc.target", "grpc.lb.rls.server_target")); + EXPECT_THAT(descriptor->optional_label_keys, ::testing::ElementsAre()); +} + +TEST_F(RlsMetricsEnd2endTest, MetricDefinitionCacheEntries) { + const auto* descriptor = + grpc_core::GlobalInstrumentsRegistryTestPeer::FindMetricDescriptorByName( + "grpc.lb.rls.cache_entries"); + ASSERT_NE(descriptor, nullptr); + EXPECT_EQ(descriptor->value_type, + grpc_core::GlobalInstrumentsRegistry::ValueType::kInt64); + EXPECT_EQ( + descriptor->instrument_type, + grpc_core::GlobalInstrumentsRegistry::InstrumentType::kCallbackGauge); + EXPECT_EQ(descriptor->enable_by_default, false); + EXPECT_EQ(descriptor->name, "grpc.lb.rls.cache_entries"); + EXPECT_EQ(descriptor->unit, "{entry}"); + EXPECT_THAT(descriptor->label_keys, + ::testing::ElementsAre("grpc.target", "grpc.lb.rls.server_target", + "grpc.lb.rls.instance_uuid")); + EXPECT_THAT(descriptor->optional_label_keys, ::testing::ElementsAre()); +} + +TEST_F(RlsMetricsEnd2endTest, MetricDefinitionCacheSize) { + const auto* descriptor = + grpc_core::GlobalInstrumentsRegistryTestPeer::FindMetricDescriptorByName( + "grpc.lb.rls.cache_size"); + ASSERT_NE(descriptor, nullptr); + EXPECT_EQ(descriptor->value_type, + grpc_core::GlobalInstrumentsRegistry::ValueType::kInt64); + EXPECT_EQ( + descriptor->instrument_type, + grpc_core::GlobalInstrumentsRegistry::InstrumentType::kCallbackGauge); + EXPECT_EQ(descriptor->enable_by_default, false); + EXPECT_EQ(descriptor->name, "grpc.lb.rls.cache_size"); + EXPECT_EQ(descriptor->unit, "By"); + EXPECT_THAT(descriptor->label_keys, + ::testing::ElementsAre("grpc.target", "grpc.lb.rls.server_target", + "grpc.lb.rls.instance_uuid")); + EXPECT_THAT(descriptor->optional_label_keys, ::testing::ElementsAre()); +} + +TEST_F(RlsMetricsEnd2endTest, MetricValues) { + auto kMetricTargetPicks = + grpc_core::GlobalInstrumentsRegistryTestPeer:: + FindUInt64CounterHandleByName("grpc.lb.rls.target_picks") + .value(); + auto kMetricFailedPicks = + grpc_core::GlobalInstrumentsRegistryTestPeer:: + FindUInt64CounterHandleByName("grpc.lb.rls.failed_picks") + .value(); + auto kMetricCacheEntries = + grpc_core::GlobalInstrumentsRegistryTestPeer:: + FindCallbackInt64GaugeHandleByName("grpc.lb.rls.cache_entries") + .value(); + auto kMetricCacheSize = + grpc_core::GlobalInstrumentsRegistryTestPeer:: + FindCallbackInt64GaugeHandleByName("grpc.lb.rls.cache_size") + .value(); + StartBackends(2); + SetNextResolution( + MakeServiceConfigBuilder() + .AddKeyBuilder(absl::StrFormat("\"names\":[{" + " \"service\":\"%s\"," + " \"method\":\"%s\"" + "}]," + "\"headers\":[" + " {" + " \"key\":\"%s\"," + " \"names\":[" + " \"key1\"" + " ]" + " }" + "]", + kServiceValue, kMethodValue, kTestKey)) + .Build()); + const std::string rls_target0 = grpc_core::LocalIpUri(backends_[0]->port_); + const std::string rls_target1 = grpc_core::LocalIpUri(backends_[1]->port_); + // Send an RPC to the target for backend 0. + rls_server_->service_.SetResponse(BuildRlsRequest({{kTestKey, rls_target0}}), + BuildRlsResponse({rls_target0})); + CheckRpcSendOk(DEBUG_LOCATION, + RpcOptions().set_metadata({{"key1", rls_target0}})); + EXPECT_EQ(rls_server_->service_.request_count(), 1); + EXPECT_EQ(rls_server_->service_.response_count(), 1); + EXPECT_EQ(backends_[0]->service_.request_count(), 1); + EXPECT_EQ(backends_[1]->service_.request_count(), 0); + // Check exported metrics. + EXPECT_THAT( + stats_plugin_->GetCounterValue( + kMetricTargetPicks, + {target_uri_, rls_server_target_, rls_target0, "complete"}, {}), + ::testing::Optional(1)); + EXPECT_THAT( + stats_plugin_->GetCounterValue( + kMetricTargetPicks, + {target_uri_, rls_server_target_, rls_target1, "complete"}, {}), + absl::nullopt); + EXPECT_EQ(stats_plugin_->GetCounterValue( + kMetricFailedPicks, {target_uri_, rls_server_target_}, {}), + absl::nullopt); + stats_plugin_->TriggerCallbacks(); + EXPECT_THAT(stats_plugin_->GetCallbackGaugeValue( + kMetricCacheEntries, + {target_uri_, rls_server_target_, kRlsInstanceUuid}, {}), + ::testing::Optional(1)); + auto cache_size = stats_plugin_->GetCallbackGaugeValue( + kMetricCacheSize, {target_uri_, rls_server_target_, kRlsInstanceUuid}, + {}); + EXPECT_THAT(cache_size, ::testing::Optional(::testing::Ge(1))); + // Send an RPC to the target for backend 1. + rls_server_->service_.SetResponse(BuildRlsRequest({{kTestKey, rls_target1}}), + BuildRlsResponse({rls_target1})); + CheckRpcSendOk(DEBUG_LOCATION, + RpcOptions().set_metadata({{"key1", rls_target1}})); + EXPECT_EQ(rls_server_->service_.request_count(), 2); + EXPECT_EQ(rls_server_->service_.response_count(), 2); + EXPECT_EQ(backends_[0]->service_.request_count(), 1); + EXPECT_EQ(backends_[1]->service_.request_count(), 1); + // Check exported metrics. + EXPECT_THAT( + stats_plugin_->GetCounterValue( + kMetricTargetPicks, + {target_uri_, rls_server_target_, rls_target0, "complete"}, {}), + ::testing::Optional(1)); + EXPECT_THAT( + stats_plugin_->GetCounterValue( + kMetricTargetPicks, + {target_uri_, rls_server_target_, rls_target1, "complete"}, {}), + ::testing::Optional(1)); + EXPECT_EQ(stats_plugin_->GetCounterValue( + kMetricFailedPicks, {target_uri_, rls_server_target_}, {}), + absl::nullopt); + stats_plugin_->TriggerCallbacks(); + EXPECT_THAT(stats_plugin_->GetCallbackGaugeValue( + kMetricCacheEntries, + {target_uri_, rls_server_target_, kRlsInstanceUuid}, {}), + ::testing::Optional(2)); + auto cache_size2 = stats_plugin_->GetCallbackGaugeValue( + kMetricCacheSize, {target_uri_, rls_server_target_, kRlsInstanceUuid}, + {}); + EXPECT_THAT(cache_size2, ::testing::Optional(::testing::Ge(2))); + if (cache_size.has_value() && cache_size2.has_value()) { + EXPECT_GT(*cache_size2, *cache_size); + } + // Send an RPC for which the RLS server has no response, which means + // that the RLS request will fail. There is no default target, so the + // data plane RPC will fail. + CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::UNAVAILABLE, + "RLS request failed: INTERNAL: no response entry", + RpcOptions().set_metadata({{"key1", kTestValue}})); + EXPECT_THAT( + rls_server_->service_.GetUnmatchedRequests(), + ::testing::ElementsAre( + // TODO(roth): Change this to use ::testing::ProtoEquals() + // once that becomes available in OSS. + ::testing::Property( + &RouteLookupRequest::DebugString, + BuildRlsRequest({{kTestKey, kTestValue}}).DebugString()))); + EXPECT_EQ(rls_server_->service_.request_count(), 3); + EXPECT_EQ(rls_server_->service_.response_count(), 2); + EXPECT_EQ(backends_[0]->service_.request_count(), 1); + EXPECT_EQ(backends_[1]->service_.request_count(), 1); + // Check exported metrics. + EXPECT_THAT( + stats_plugin_->GetCounterValue( + kMetricTargetPicks, + {target_uri_, rls_server_target_, rls_target0, "complete"}, {}), + ::testing::Optional(1)); + EXPECT_THAT( + stats_plugin_->GetCounterValue( + kMetricTargetPicks, + {target_uri_, rls_server_target_, rls_target1, "complete"}, {}), + ::testing::Optional(1)); + EXPECT_THAT(stats_plugin_->GetCounterValue( + kMetricFailedPicks, {target_uri_, rls_server_target_}, {}), + ::testing::Optional(1)); + stats_plugin_->TriggerCallbacks(); + EXPECT_THAT(stats_plugin_->GetCallbackGaugeValue( + kMetricCacheEntries, + {target_uri_, rls_server_target_, kRlsInstanceUuid}, {}), + ::testing::Optional(3)); + auto cache_size3 = stats_plugin_->GetCallbackGaugeValue( + kMetricCacheSize, {target_uri_, rls_server_target_, kRlsInstanceUuid}, + {}); + EXPECT_THAT(cache_size3, ::testing::Optional(::testing::Ge(3))); + if (cache_size.has_value() && cache_size3.has_value()) { + EXPECT_GT(*cache_size3, *cache_size); + } +} + +TEST_F(RlsMetricsEnd2endTest, MetricValuesDefaultTargetRpcs) { + auto kMetricDefaultTargetPicks = + grpc_core::GlobalInstrumentsRegistryTestPeer:: + FindUInt64CounterHandleByName("grpc.lb.rls.default_target_picks") + .value(); + StartBackends(1); + const std::string default_target = grpc_core::LocalIpUri(backends_[0]->port_); + SetNextResolution( + MakeServiceConfigBuilder() + .AddKeyBuilder(absl::StrFormat("\"names\":[{" + " \"service\":\"%s\"," + " \"method\":\"%s\"" + "}]," + "\"headers\":[" + " {" + " \"key\":\"%s\"," + " \"names\":[" + " \"key1\"" + " ]" + " }" + "]", + kServiceValue, kMethodValue, kTestKey)) + .set_default_target(default_target) + .Build()); + // Don't give the RLS server a response, so the RLS request will fail. + // The data plane RPC should be sent to the default target. + CheckRpcSendOk(DEBUG_LOCATION, + RpcOptions().set_metadata({{"key1", kTestValue}})); + EXPECT_THAT( + rls_server_->service_.GetUnmatchedRequests(), + ::testing::ElementsAre( + // TODO(roth): Change this to use ::testing::ProtoEquals() + // once that becomes available in OSS. + ::testing::Property( + &RouteLookupRequest::DebugString, + BuildRlsRequest({{kTestKey, kTestValue}}).DebugString()))); + EXPECT_EQ(rls_server_->service_.request_count(), 1); + EXPECT_EQ(rls_server_->service_.response_count(), 0); + EXPECT_EQ(backends_[0]->service_.request_count(), 1); + // Check expected metrics. + EXPECT_THAT( + stats_plugin_->GetCounterValue( + kMetricDefaultTargetPicks, + {target_uri_, rls_server_target_, default_target, "complete"}, {}), + ::testing::Optional(1)); +} + } // namespace } // namespace testing } // namespace grpc diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index ae04b133f2f..1fbf624c5d3 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -2398,6 +2398,8 @@ src/core/lib/gprpp/time_util.cc \ src/core/lib/gprpp/time_util.h \ src/core/lib/gprpp/type_list.h \ src/core/lib/gprpp/unique_type_name.h \ +src/core/lib/gprpp/uuid_v4.cc \ +src/core/lib/gprpp/uuid_v4.h \ src/core/lib/gprpp/validation_errors.cc \ src/core/lib/gprpp/validation_errors.h \ src/core/lib/gprpp/windows/directory_reader.cc \ @@ -2889,6 +2891,7 @@ src/core/load_balancing/priority/priority.cc \ src/core/load_balancing/ring_hash/ring_hash.cc \ src/core/load_balancing/ring_hash/ring_hash.h \ src/core/load_balancing/rls/rls.cc \ +src/core/load_balancing/rls/rls.h \ src/core/load_balancing/round_robin/round_robin.cc \ src/core/load_balancing/subchannel_interface.h \ src/core/load_balancing/subchannel_list.h \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 5990bd20e58..9f4eb9b2777 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -2172,6 +2172,8 @@ src/core/lib/gprpp/time_util.cc \ src/core/lib/gprpp/time_util.h \ src/core/lib/gprpp/type_list.h \ src/core/lib/gprpp/unique_type_name.h \ +src/core/lib/gprpp/uuid_v4.cc \ +src/core/lib/gprpp/uuid_v4.h \ src/core/lib/gprpp/validation_errors.cc \ src/core/lib/gprpp/validation_errors.h \ src/core/lib/gprpp/windows/directory_reader.cc \ @@ -2666,6 +2668,7 @@ src/core/load_balancing/priority/priority.cc \ src/core/load_balancing/ring_hash/ring_hash.cc \ src/core/load_balancing/ring_hash/ring_hash.h \ src/core/load_balancing/rls/rls.cc \ +src/core/load_balancing/rls/rls.h \ src/core/load_balancing/round_robin/round_robin.cc \ src/core/load_balancing/subchannel_interface.h \ src/core/load_balancing/subchannel_list.h \ From 427c8a89e97e2df0f014c8b97ef43151b82826e3 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 4 Mar 2024 14:04:20 -0800 Subject: [PATCH 03/52] [chaotic-good] Add a microbenchmark for ping pong round trips (#36050) also: - remove tail recursion from promise endpoint read completion (actually overflowed stack!) - remove retry filter from benchmark - we probably don't want this long term, but for now nobody else is using this benchmark and our use case doesn't use grpc retries so.... good enough Closes #36050 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/36050 from ctiller:cgbm 65b1c267675035aad5f1721b57cbc4c65c0de6e1 PiperOrigin-RevId: 612577071 --- src/core/lib/transport/promise_endpoint.cc | 79 ++++++------ test/cpp/microbenchmarks/BUILD | 17 +++ ..._fullstack_unary_ping_pong_chaotic_good.cc | 122 ++++++++++++++++++ test/cpp/microbenchmarks/fullstack_fixtures.h | 2 + 4 files changed, 182 insertions(+), 38 deletions(-) create mode 100644 test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong_chaotic_good.cc diff --git a/src/core/lib/transport/promise_endpoint.cc b/src/core/lib/transport/promise_endpoint.cc index 1fd08b8917e..5159b2dc6d4 100644 --- a/src/core/lib/transport/promise_endpoint.cc +++ b/src/core/lib/transport/promise_endpoint.cc @@ -58,50 +58,53 @@ PromiseEndpoint::GetLocalAddress() const { } void PromiseEndpoint::ReadState::Complete(absl::Status status, - size_t num_bytes_requested) { - if (!status.ok()) { - // Invalidates all previous reads. - pending_buffer.Clear(); - buffer.Clear(); + const size_t num_bytes_requested) { + while (true) { + if (!status.ok()) { + // Invalidates all previous reads. + pending_buffer.Clear(); + buffer.Clear(); + result = status; + auto w = std::move(waker); + complete.store(true, std::memory_order_release); + w.Wakeup(); + return; + } + // Appends `pending_buffer` to `buffer`. + pending_buffer.MoveFirstNBytesIntoSliceBuffer(pending_buffer.Length(), + buffer); + GPR_DEBUG_ASSERT(pending_buffer.Count() == 0u); + if (buffer.Length() < num_bytes_requested) { + // A further read is needed. + // Set read args with number of bytes needed as hint. + grpc_event_engine::experimental::EventEngine::Endpoint::ReadArgs + read_args = { + static_cast(num_bytes_requested - buffer.Length())}; + // If `Read()` returns true immediately, the callback will not be + // called. We still need to call our callback to pick up the result and + // maybe do further reads. + auto ep = endpoint.lock(); + if (ep == nullptr) { + status = absl::UnavailableError("Endpoint closed during read."); + continue; + } + if (ep->Read( + [self = Ref(), num_bytes_requested](absl::Status status) { + ApplicationCallbackExecCtx callback_exec_ctx; + ExecCtx exec_ctx; + self->Complete(std::move(status), num_bytes_requested); + }, + &pending_buffer, &read_args)) { + continue; + } + return; + } result = status; auto w = std::move(waker); complete.store(true, std::memory_order_release); w.Wakeup(); return; } - // Appends `pending_buffer` to `buffer`. - pending_buffer.MoveFirstNBytesIntoSliceBuffer(pending_buffer.Length(), - buffer); - GPR_DEBUG_ASSERT(pending_buffer.Count() == 0u); - if (buffer.Length() < num_bytes_requested) { - // A further read is needed. - // Set read args with number of bytes needed as hint. - grpc_event_engine::experimental::EventEngine::Endpoint::ReadArgs read_args = - {static_cast(num_bytes_requested - buffer.Length())}; - // If `Read()` returns true immediately, the callback will not be - // called. We still need to call our callback to pick up the result and - // maybe do further reads. - auto ep = endpoint.lock(); - if (ep == nullptr) { - Complete(absl::UnavailableError("Endpoint closed during read."), - num_bytes_requested); - return; - } - if (ep->Read( - [self = Ref(), num_bytes_requested](absl::Status status) { - ApplicationCallbackExecCtx callback_exec_ctx; - ExecCtx exec_ctx; - self->Complete(std::move(status), num_bytes_requested); - }, - &pending_buffer, &read_args)) { - Complete(std::move(status), num_bytes_requested); - } - return; - } - result = status; - auto w = std::move(waker); - complete.store(true, std::memory_order_release); - w.Wakeup(); } } // namespace grpc_core diff --git a/test/cpp/microbenchmarks/BUILD b/test/cpp/microbenchmarks/BUILD index b54c5258847..b5def4b13f8 100644 --- a/test/cpp/microbenchmarks/BUILD +++ b/test/cpp/microbenchmarks/BUILD @@ -323,6 +323,23 @@ grpc_cc_test( deps = [":fullstack_unary_ping_pong_h"], ) +grpc_cc_test( + name = "bm_fullstack_unary_ping_pong_chaotic_good", + size = "large", + srcs = [ + "bm_fullstack_unary_ping_pong_chaotic_good.cc", + ], + args = grpc_benchmark_args(), + tags = [ + "no_mac", # to emulate "excluded_poll_engines: poll" + "no_windows", + ], + deps = [ + ":fullstack_unary_ping_pong_h", + "//:grpcpp_chaotic_good", + ], +) + grpc_cc_test( name = "bm_chttp2_hpack", srcs = ["bm_chttp2_hpack.cc"], diff --git a/test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong_chaotic_good.cc b/test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong_chaotic_good.cc new file mode 100644 index 00000000000..01a0b719b4b --- /dev/null +++ b/test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong_chaotic_good.cc @@ -0,0 +1,122 @@ +// +// +// Copyright 2024 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// + +// Benchmark gRPC end2end in various configurations for chaotic good +// TODO(ctiller): fold back into bm_fullstack_unary_ping_pong.cc once chaotic +// good can run without custom experiment configuration. + +#include "src/cpp/ext/chaotic_good.h" +#include "test/core/util/test_config.h" +#include "test/cpp/microbenchmarks/fullstack_unary_ping_pong.h" +#include "test/cpp/util/test_config.h" + +namespace grpc { +namespace testing { + +class ChaoticGoodFixture : public BaseFixture { + public: + explicit ChaoticGoodFixture( + Service* service, + const FixtureConfiguration& config = FixtureConfiguration()) { + auto address = MakeAddress(&port_); + ServerBuilder b; + if (address.length() > 0) { + b.AddListeningPort(address, ChaoticGoodInsecureServerCredentials()); + } + cq_ = b.AddCompletionQueue(true); + b.RegisterService(service); + config.ApplyCommonServerBuilderConfig(&b); + server_ = b.BuildAndStart(); + ChannelArguments args; + config.ApplyCommonChannelArguments(&args); + if (address.length() > 0) { + channel_ = grpc::CreateCustomChannel( + address, ChaoticGoodInsecureChannelCredentials(), args); + } else { + channel_ = server_->InProcessChannel(args); + } + } + + ~ChaoticGoodFixture() override { + server_->Shutdown(grpc_timeout_milliseconds_to_deadline(0)); + cq_->Shutdown(); + void* tag; + bool ok; + while (cq_->Next(&tag, &ok)) { + } + grpc_recycle_unused_port(port_); + } + + ServerCompletionQueue* cq() { return cq_.get(); } + std::shared_ptr channel() { return channel_; } + + private: + static std::string MakeAddress(int* port) { + *port = grpc_pick_unused_port_or_die(); + std::stringstream addr; + addr << "localhost:" << *port; + return addr.str(); + } + + std::unique_ptr server_; + std::unique_ptr cq_; + std::shared_ptr channel_; + int port_; +}; + +//****************************************************************************** +// CONFIGURATIONS +// + +// Replace "benchmark::internal::Benchmark" with "::testing::Benchmark" to use +// internal microbenchmarking tooling +static void SweepSizesArgs(benchmark::internal::Benchmark* b) { + b->Args({0, 0}); + for (int i = 1; i <= 128 * 1024 * 1024; i *= 8) { + b->Args({i, 0}); + b->Args({0, i}); + b->Args({i, i}); + } +} + +BENCHMARK_TEMPLATE(BM_UnaryPingPong, ChaoticGoodFixture, NoOpMutator, + NoOpMutator) + ->Apply(SweepSizesArgs); + +} // namespace testing +} // namespace grpc + +// Some distros have RunSpecifiedBenchmarks under the benchmark namespace, +// and others do not. This allows us to support both modes. +namespace benchmark { +void RunTheBenchmarksNamespaced() { RunSpecifiedBenchmarks(); } +} // namespace benchmark + +int main(int argc, char** argv) { + grpc_core::ForceEnableExperiment("event_engine_client", true); + grpc_core::ForceEnableExperiment("event_engine_listener", true); + grpc_core::ForceEnableExperiment("promise_based_client_call", true); + grpc_core::ForceEnableExperiment("promise_based_server_call", true); + grpc_core::ForceEnableExperiment("chaotic_good", true); + grpc::testing::TestEnvironment env(&argc, argv); + LibraryInitializer libInit; + ::benchmark::Initialize(&argc, argv); + grpc::testing::InitTest(&argc, &argv, false); + benchmark::RunTheBenchmarksNamespaced(); + return 0; +} diff --git a/test/cpp/microbenchmarks/fullstack_fixtures.h b/test/cpp/microbenchmarks/fullstack_fixtures.h index e2f9cfefc6d..40f1a032906 100644 --- a/test/cpp/microbenchmarks/fullstack_fixtures.h +++ b/test/cpp/microbenchmarks/fullstack_fixtures.h @@ -19,6 +19,7 @@ #ifndef GRPC_TEST_CPP_MICROBENCHMARKS_FULLSTACK_FIXTURES_H #define GRPC_TEST_CPP_MICROBENCHMARKS_FULLSTACK_FIXTURES_H +#include #include #include #include @@ -54,6 +55,7 @@ class FixtureConfiguration { virtual void ApplyCommonChannelArguments(ChannelArguments* c) const { c->SetInt(GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH, INT_MAX); c->SetInt(GRPC_ARG_MAX_SEND_MESSAGE_LENGTH, INT_MAX); + c->SetInt(GRPC_ARG_ENABLE_RETRIES, 0); c->SetResourceQuota(ResourceQuota()); } From 672d8abdca6e7fb4919586bbc4e7fe0212a68abc Mon Sep 17 00:00:00 2001 From: Xuan Wang Date: Mon, 4 Mar 2024 17:04:23 -0800 Subject: [PATCH 04/52] [Python Otel] Allow start observability without context manager (#35932) Allow start observability globally with a new API `start_open_telemetry_observability`. Closes #35932 PiperOrigin-RevId: 612639020 --- .../observability_greeter_client.py | 20 ++-- .../observability_greeter_server.py | 34 ++++--- .../grpc_observability/__init__.py | 13 ++- .../_open_telemetry_observability.py | 59 +++++++++-- .../observability/_observability_api_test.py | 2 + .../_open_telemetry_observability_test.py | 99 ++++++++++++++++++- 6 files changed, 190 insertions(+), 37 deletions(-) diff --git a/examples/python/observability/observability_greeter_client.py b/examples/python/observability/observability_greeter_client.py index 25f7d68a4c8..5e0e024f93f 100644 --- a/examples/python/observability/observability_greeter_client.py +++ b/examples/python/observability/observability_greeter_client.py @@ -47,16 +47,16 @@ def run(): provider = MeterProvider(metric_readers=[reader]) otel_plugin = BaseOpenTelemetryPlugin(provider) - with grpc_observability.OpenTelemetryObservability(plugins=[otel_plugin]): - with grpc.insecure_channel(target="localhost:50051") as channel: - stub = helloworld_pb2_grpc.GreeterStub(channel) - try: - response = stub.SayHello( - helloworld_pb2.HelloRequest(name="You") - ) - print(f"Greeter client received: {response.message}") - except grpc.RpcError as rpc_error: - print("Call failed with code: ", rpc_error.code()) + grpc_observability.start_open_telemetry_observability(plugins=[otel_plugin]) + + with grpc.insecure_channel(target="localhost:50051") as channel: + stub = helloworld_pb2_grpc.GreeterStub(channel) + try: + response = stub.SayHello(helloworld_pb2.HelloRequest(name="You")) + print(f"Greeter client received: {response.message}") + except grpc.RpcError as rpc_error: + print("Call failed with code: ", rpc_error.code()) + grpc_observability.end_open_telemetry_observability() # Sleep to make sure all metrics are exported. time.sleep(5) diff --git a/examples/python/observability/observability_greeter_server.py b/examples/python/observability/observability_greeter_server.py index 1f5a9689be6..542e68be42d 100644 --- a/examples/python/observability/observability_greeter_server.py +++ b/examples/python/observability/observability_greeter_server.py @@ -57,22 +57,24 @@ def serve(): provider = MeterProvider(metric_readers=[reader]) otel_plugin = BaseOpenTelemetryPlugin(provider) - with grpc_observability.OpenTelemetryObservability(plugins=[otel_plugin]): - server = grpc.server( - thread_pool=futures.ThreadPoolExecutor(max_workers=10), - ) - helloworld_pb2_grpc.add_GreeterServicer_to_server(Greeter(), server) - server.add_insecure_port("[::]:" + _SERVER_PORT) - server.start() - print("Server started, listening on " + _SERVER_PORT) - - # Sleep to make sure client made RPC call and all metrics are exported. - time.sleep(10) - print("Metrics exported on Server side:") - for metric in all_metrics: - print(metric) - - server.stop(0) + grpc_observability.start_open_telemetry_observability(plugins=[otel_plugin]) + + server = grpc.server( + thread_pool=futures.ThreadPoolExecutor(max_workers=10), + ) + helloworld_pb2_grpc.add_GreeterServicer_to_server(Greeter(), server) + server.add_insecure_port("[::]:" + _SERVER_PORT) + server.start() + print("Server started, listening on " + _SERVER_PORT) + + # Sleep to make sure client made RPC call and all metrics are exported. + time.sleep(10) + print("Metrics exported on Server side:") + for metric in all_metrics: + print(metric) + + server.stop(0) + grpc_observability.end_open_telemetry_observability() if __name__ == "__main__": diff --git a/src/python/grpcio_observability/grpc_observability/__init__.py b/src/python/grpcio_observability/grpc_observability/__init__.py index 7dad3d83efe..f4a7ef453e9 100644 --- a/src/python/grpcio_observability/grpc_observability/__init__.py +++ b/src/python/grpcio_observability/grpc_observability/__init__.py @@ -15,6 +15,17 @@ from grpc_observability._open_telemetry_observability import ( OpenTelemetryObservability, ) +from grpc_observability._open_telemetry_observability import ( + end_open_telemetry_observability, +) +from grpc_observability._open_telemetry_observability import ( + start_open_telemetry_observability, +) from grpc_observability._open_telemetry_plugin import OpenTelemetryPlugin -__all__ = ("OpenTelemetryObservability", "OpenTelemetryPlugin") +__all__ = ( + "OpenTelemetryObservability", + "OpenTelemetryPlugin", + "start_open_telemetry_observability", + "end_open_telemetry_observability", +) diff --git a/src/python/grpcio_observability/grpc_observability/_open_telemetry_observability.py b/src/python/grpcio_observability/grpc_observability/_open_telemetry_observability.py index a102509e5c8..74f40b66a1f 100644 --- a/src/python/grpcio_observability/grpc_observability/_open_telemetry_observability.py +++ b/src/python/grpcio_observability/grpc_observability/_open_telemetry_observability.py @@ -13,6 +13,7 @@ # limitations under the License. import logging +import threading import time from typing import Any, Iterable, Optional @@ -54,6 +55,22 @@ GRPC_STATUS_CODE_TO_STRING = { grpc.StatusCode.DATA_LOSS: "DATA_LOSS", } +_observability_lock: threading.RLock = threading.RLock() +_OPEN_TELEMETRY_OBSERVABILITY: Optional["OpenTelemetryObservability"] = None + + +def start_open_telemetry_observability( + *, + plugins: Optional[Iterable[OpenTelemetryPlugin]] = None, +) -> None: + _start_open_telemetry_observability( + OpenTelemetryObservability(plugins=plugins) + ) + + +def end_open_telemetry_observability() -> None: + _end_open_telemetry_observability() + # pylint: disable=no-self-use class OpenTelemetryObservability(grpc._observability.ObservabilityPlugin): @@ -66,7 +83,6 @@ class OpenTelemetryObservability(grpc._observability.ObservabilityPlugin): """ exporter: "grpc_observability.Exporter" - plugins: Iterable[OpenTelemetryPlugin] def __init__( self, @@ -80,13 +96,20 @@ class OpenTelemetryObservability(grpc._observability.ObservabilityPlugin): self.exporter = _OpenTelemetryExporterDelegator(_plugins) + def __enter__(self): + _start_open_telemetry_observability(self) + return self + + def __exit__(self, exc_type, exc_val, exc_tb) -> None: + _end_open_telemetry_observability() + + def observability_init(self): try: _cyobservability.activate_stats() self.set_stats(True) except Exception as e: # pylint: disable=broad-except raise ValueError(f"Activate observability metrics failed with: {e}") - def __enter__(self): try: _cyobservability.cyobservability_init(self.exporter) # TODO(xuanwn): Use specific exceptons @@ -94,12 +117,8 @@ class OpenTelemetryObservability(grpc._observability.ObservabilityPlugin): _LOGGER.exception("Initiate observability failed with: %s", e) grpc._observability.observability_init(self) - return self - def __exit__(self, exc_type, exc_val, exc_tb) -> None: - self.exit() - - def exit(self) -> None: + def observability_deinit(self) -> None: # Sleep so we don't loss any data. If we shutdown export thread # immediately after exit, it's possible that core didn't call RecordEnd # in callTracer, and all data recorded by calling RecordEnd will be @@ -150,3 +169,29 @@ class OpenTelemetryObservability(grpc._observability.ObservabilityPlugin): _cyobservability._record_rpc_latency( self.exporter, method, target, rpc_latency, status_code ) + + +def _start_open_telemetry_observability( + otel_o11y: OpenTelemetryObservability, +) -> None: + global _OPEN_TELEMETRY_OBSERVABILITY # pylint: disable=global-statement + with _observability_lock: + if _OPEN_TELEMETRY_OBSERVABILITY is None: + _OPEN_TELEMETRY_OBSERVABILITY = otel_o11y + _OPEN_TELEMETRY_OBSERVABILITY.observability_init() + else: + raise RuntimeError( + "gPRC Python observability was already initialized!" + ) + + +def _end_open_telemetry_observability() -> None: + global _OPEN_TELEMETRY_OBSERVABILITY # pylint: disable=global-statement + with _observability_lock: + if not _OPEN_TELEMETRY_OBSERVABILITY: + raise RuntimeError( + "Trying to end gPRC Python observability without initialize first!" + ) + else: + _OPEN_TELEMETRY_OBSERVABILITY.observability_deinit() + _OPEN_TELEMETRY_OBSERVABILITY = None diff --git a/src/python/grpcio_tests/tests/observability/_observability_api_test.py b/src/python/grpcio_tests/tests/observability/_observability_api_test.py index efdd63dca0a..e3f92338de5 100644 --- a/src/python/grpcio_tests/tests/observability/_observability_api_test.py +++ b/src/python/grpcio_tests/tests/observability/_observability_api_test.py @@ -24,6 +24,8 @@ class AllTest(unittest.TestCase): expected_observability_code_elements = ( "OpenTelemetryObservability", "OpenTelemetryPlugin", + "start_open_telemetry_observability", + "end_open_telemetry_observability", ) self.assertCountEqual( diff --git a/src/python/grpcio_tests/tests/observability/_open_telemetry_observability_test.py b/src/python/grpcio_tests/tests/observability/_open_telemetry_observability_test.py index 81c9750970f..7031965b49f 100644 --- a/src/python/grpcio_tests/tests/observability/_open_telemetry_observability_test.py +++ b/src/python/grpcio_tests/tests/observability/_open_telemetry_observability_test.py @@ -20,6 +20,7 @@ import sys import time from typing import Any, Callable, Dict, List, Optional, Set import unittest +from unittest.mock import patch import grpc import grpc_observability @@ -134,7 +135,7 @@ class OpenTelemetryObservabilityTest(unittest.TestCase): if self._server: self._server.stop(0) - def testRecordUnaryUnary(self): + def testRecordUnaryUnaryUseContextManager(self): otel_plugin = BaseTestOpenTelemetryPlugin(self._provider) with grpc_observability.OpenTelemetryObservability( plugins=[otel_plugin] @@ -146,6 +147,78 @@ class OpenTelemetryObservabilityTest(unittest.TestCase): self._validate_metrics_exist(self.all_metrics) self._validate_all_metrics_names(self.all_metrics) + def testRecordUnaryUnaryUseGlobalInit(self): + otel_plugin = BaseTestOpenTelemetryPlugin(self._provider) + + grpc_observability.start_open_telemetry_observability( + plugins=[otel_plugin] + ) + server, port = _test_server.start_server() + self._server = server + _test_server.unary_unary_call(port=port) + + self._validate_metrics_exist(self.all_metrics) + self._validate_all_metrics_names(self.all_metrics) + grpc_observability.end_open_telemetry_observability() + + def testCallGlobalInitThrowErrorWhenGlobalCalled(self): + grpc_observability.start_open_telemetry_observability(plugins=[]) + try: + grpc_observability.start_open_telemetry_observability(plugins=[]) + except RuntimeError as exp: + self.assertIn( + "gPRC Python observability was already initialized", str(exp) + ) + + grpc_observability.end_open_telemetry_observability() + + def testCallGlobalInitThrowErrorWhenContextManagerCalled(self): + with grpc_observability.OpenTelemetryObservability(plugins=[]): + try: + grpc_observability.start_open_telemetry_observability( + plugins=[] + ) + except RuntimeError as exp: + self.assertIn( + "gPRC Python observability was already initialized", + str(exp), + ) + + def testCallContextManagerThrowErrorWhenGlobalInitCalled(self): + grpc_observability.start_open_telemetry_observability(plugins=[]) + try: + with grpc_observability.OpenTelemetryObservability(plugins=[]): + pass + except RuntimeError as exp: + self.assertIn( + "gPRC Python observability was already initialized", str(exp) + ) + grpc_observability.end_open_telemetry_observability() + + def testContextManagerThrowErrorWhenContextManagerCalled(self): + with grpc_observability.OpenTelemetryObservability(plugins=[]): + try: + with grpc_observability.OpenTelemetryObservability(plugins=[]): + pass + except RuntimeError as exp: + self.assertIn( + "gPRC Python observability was already initialized", + str(exp), + ) + + def testNoErrorCallGlobalInitThenContextManager(self): + grpc_observability.start_open_telemetry_observability(plugins=[]) + grpc_observability.end_open_telemetry_observability() + + with grpc_observability.OpenTelemetryObservability(plugins=[]): + pass + + def testNoErrorCallContextManagerThenGlobalInit(self): + with grpc_observability.OpenTelemetryObservability(plugins=[]): + pass + grpc_observability.start_open_telemetry_observability(plugins=[]) + grpc_observability.end_open_telemetry_observability() + def testRecordUnaryUnaryWithClientInterceptor(self): interceptor = _ClientUnaryUnaryInterceptor() otel_plugin = BaseTestOpenTelemetryPlugin(self._provider) @@ -212,7 +285,7 @@ class OpenTelemetryObservabilityTest(unittest.TestCase): self._validate_metrics_exist(self.all_metrics) self._validate_all_metrics_names(self.all_metrics) - def testNoRecordAfterExit(self): + def testNoRecordAfterExitUseContextManager(self): otel_plugin = BaseTestOpenTelemetryPlugin(self._provider) with grpc_observability.OpenTelemetryObservability( plugins=[otel_plugin] @@ -230,6 +303,26 @@ class OpenTelemetryObservabilityTest(unittest.TestCase): with self.assertRaisesRegex(AssertionError, "No metrics was exported"): self._validate_metrics_exist(self.all_metrics) + def testNoRecordAfterExitUseGlobal(self): + otel_plugin = BaseTestOpenTelemetryPlugin(self._provider) + + grpc_observability.start_open_telemetry_observability( + plugins=[otel_plugin] + ) + server, port = _test_server.start_server() + self._server = server + self._port = port + _test_server.unary_unary_call(port=port) + grpc_observability.end_open_telemetry_observability() + + self._validate_metrics_exist(self.all_metrics) + self._validate_all_metrics_names(self.all_metrics) + + self.all_metrics = defaultdict(list) + _test_server.unary_unary_call(port=self._port) + with self.assertRaisesRegex(AssertionError, "No metrics was exported"): + self._validate_metrics_exist(self.all_metrics) + def testRecordUnaryStream(self): otel_plugin = BaseTestOpenTelemetryPlugin(self._provider) @@ -342,7 +435,7 @@ class OpenTelemetryObservabilityTest(unittest.TestCase): message: Optional[Callable[[], str]] = None, ) -> None: message = message or (lambda: "Proposition did not evaluate to true") - timeout = timeout or datetime.timedelta(seconds=10) + timeout = timeout or datetime.timedelta(seconds=5) end = datetime.datetime.now() + timeout while datetime.datetime.now() < end: if predicate(): From 692b7fcb7f63b1b3634fb52d490c04f95bdaa288 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Mon, 4 Mar 2024 17:29:50 -0800 Subject: [PATCH 05/52] [bazel] Add OTel C++ submodule to bazelignore list (#36052) --- .bazelignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.bazelignore b/.bazelignore index 6385cc6e106..7ace587d0ff 100644 --- a/.bazelignore +++ b/.bazelignore @@ -14,6 +14,7 @@ third_party/envoy-api third_party/googleapis third_party/googletest third_party/opencensus-proto +third_party/opentelemetry-cpp third_party/protobuf third_party/protoc-gen-validate third_party/re2 From db9ca10b351da70cd34359f8ea5ea00cbb4ffc1a Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Mon, 4 Mar 2024 17:41:31 -0800 Subject: [PATCH 06/52] [xDS] add optional locality label to per-call metrics (#36049) Closes #36049 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/36049 from markdroth:xds_locality_label_on_per_call_metrics 9c5aeb7fb14a229e80baf6ed267f8ead5c5fd044 PiperOrigin-RevId: 612648213 --- src/core/BUILD | 1 + src/core/ext/xds/xds_client_stats.h | 12 +- src/core/lib/channel/call_tracer.h | 5 +- .../load_balancing/xds/xds_cluster_impl.cc | 119 ++++++++++++------ .../end2end/xds/xds_cluster_end2end_test.cc | 58 ++++++--- test/cpp/end2end/xds/xds_utils.cc | 6 + test/cpp/end2end/xds/xds_utils.h | 3 + test/cpp/end2end/xds/xds_wrr_end2end_test.cc | 6 - 8 files changed, 147 insertions(+), 63 deletions(-) diff --git a/src/core/BUILD b/src/core/BUILD index 3f80297081c..f0c58279faf 100644 --- a/src/core/BUILD +++ b/src/core/BUILD @@ -5277,6 +5277,7 @@ grpc_cc_library( "lb_policy", "lb_policy_factory", "lb_policy_registry", + "match", "pollset_set", "ref_counted", "resolved_address", diff --git a/src/core/ext/xds/xds_client_stats.h b/src/core/ext/xds/xds_client_stats.h index 55385fa2e90..5f78e156768 100644 --- a/src/core/ext/xds/xds_client_stats.h +++ b/src/core/ext/xds/xds_client_stats.h @@ -63,7 +63,11 @@ class XdsLocalityName : public RefCounted { XdsLocalityName(std::string region, std::string zone, std::string sub_zone) : region_(std::move(region)), zone_(std::move(zone)), - sub_zone_(std::move(sub_zone)) {} + sub_zone_(std::move(sub_zone)), + locality_labels_( + std::make_shared>()) { + locality_labels_->emplace("grpc.lb.locality", AsHumanReadableString()); + } bool operator==(const XdsLocalityName& other) const { return region_ == other.region_ && zone_ == other.zone_ && @@ -85,6 +89,9 @@ class XdsLocalityName : public RefCounted { const std::string& region() const { return region_; } const std::string& zone() const { return zone_; } const std::string& sub_zone() const { return sub_zone_; } + std::shared_ptr> locality_labels() const { + return locality_labels_; + } std::string AsHumanReadableString() const { return absl::StrFormat("{region=\"%s\", zone=\"%s\", sub_zone=\"%s\"}", @@ -104,6 +111,7 @@ class XdsLocalityName : public RefCounted { std::string region_; std::string zone_; std::string sub_zone_; + std::shared_ptr> locality_labels_; }; // Drop stats for an xds cluster. @@ -223,6 +231,8 @@ class XdsClusterLocalityStats : public RefCounted { void AddCallFinished(const std::map* named_metrics, bool fail = false); + XdsLocalityName* locality_name() const { return name_.get(); } + private: struct Stats { std::atomic total_successful_requests{0}; diff --git a/src/core/lib/channel/call_tracer.h b/src/core/lib/channel/call_tracer.h index f189f311963..54d5e2c8508 100644 --- a/src/core/lib/channel/call_tracer.h +++ b/src/core/lib/channel/call_tracer.h @@ -128,8 +128,9 @@ class ClientCallTracer : public CallTracerAnnotationInterface { class CallAttemptTracer : public CallTracerInterface { public: enum class OptionalLabelComponent : std::uint8_t { - kXdsServiceLabels = 0, - kSize = 1, // keep last + kXdsServiceLabels, + kXdsLocalityLabels, + kSize, // keep last }; ~CallAttemptTracer() override {} diff --git a/src/core/load_balancing/xds/xds_cluster_impl.cc b/src/core/load_balancing/xds/xds_cluster_impl.cc index c4a6b1dddd8..fc5c5e32b7a 100644 --- a/src/core/load_balancing/xds/xds_cluster_impl.cc +++ b/src/core/load_balancing/xds/xds_cluster_impl.cc @@ -51,6 +51,7 @@ #include "src/core/lib/config/core_configuration.h" #include "src/core/lib/debug/trace.h" #include "src/core/lib/gprpp/debug_location.h" +#include "src/core/lib/gprpp/match.h" #include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/gprpp/ref_counted.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" @@ -191,18 +192,46 @@ class XdsClusterImplLb : public LoadBalancingPolicy { private: class StatsSubchannelWrapper : public DelegatingSubchannel { public: + // If load reporting is enabled and we have an XdsClusterLocalityStats + // object, that object already contains the locality labels. We + // need to store the locality labels directly only in the case where + // load reporting is disabled. + using LocalityData = absl::variant< + std::shared_ptr> /*locality_labels*/, + RefCountedPtr /*locality_stats*/>; + StatsSubchannelWrapper( RefCountedPtr wrapped_subchannel, - RefCountedPtr locality_stats) + LocalityData locality_data) : DelegatingSubchannel(std::move(wrapped_subchannel)), - locality_stats_(std::move(locality_stats)) {} + locality_data_(std::move(locality_data)) {} + + std::shared_ptr> locality_labels() + const { + return Match( + locality_data_, + [](const std::shared_ptr>& + locality_labels) { + return locality_labels; + }, + [](const RefCountedPtr& locality_stats) { + return locality_stats->locality_name()->locality_labels(); + }); + } XdsClusterLocalityStats* locality_stats() const { - return locality_stats_.get(); + return Match( + locality_data_, + [](const std::shared_ptr>&) { + return static_cast(nullptr); + }, + [](const RefCountedPtr& locality_stats) { + return locality_stats.get(); + }); } private: - RefCountedPtr locality_stats_; + LocalityData locality_data_; }; // A picker that wraps the picker from the child to perform drops. @@ -375,8 +404,9 @@ XdsClusterImplLb::Picker::Picker(XdsClusterImplLb* xds_cluster_impl_lb, LoadBalancingPolicy::PickResult XdsClusterImplLb::Picker::Pick( LoadBalancingPolicy::PickArgs args) { auto* call_state = static_cast(args.call_state); - if (call_state->GetCallAttemptTracer() != nullptr) { - call_state->GetCallAttemptTracer()->AddOptionalLabels( + auto* call_attempt_tracer = call_state->GetCallAttemptTracer(); + if (call_attempt_tracer != nullptr) { + call_attempt_tracer->AddOptionalLabels( OptionalLabelComponent::kXdsServiceLabels, service_labels_); } // Handle EDS drops. @@ -404,16 +434,22 @@ LoadBalancingPolicy::PickResult XdsClusterImplLb::Picker::Pick( PickResult result = picker_->Pick(args); auto* complete_pick = absl::get_if(&result.result); if (complete_pick != nullptr) { + auto* subchannel_wrapper = + static_cast(complete_pick->subchannel.get()); + // Add locality labels to per-call metrics if needed. + if (call_attempt_tracer != nullptr) { + call_attempt_tracer->AddOptionalLabels( + OptionalLabelComponent::kXdsLocalityLabels, + subchannel_wrapper->locality_labels()); + } + // Handle load reporting. RefCountedPtr locality_stats; - if (drop_stats_ != nullptr) { // If load reporting is enabled. - auto* subchannel_wrapper = - static_cast(complete_pick->subchannel.get()); - // Handle load reporting. + if (subchannel_wrapper->locality_stats() != nullptr) { locality_stats = subchannel_wrapper->locality_stats()->Ref( DEBUG_LOCATION, "SubchannelCallTracker"); - // Unwrap subchannel to pass back up the stack. - complete_pick->subchannel = subchannel_wrapper->wrapped_subchannel(); } + // Unwrap subchannel to pass back up the stack. + complete_pick->subchannel = subchannel_wrapper->wrapped_subchannel(); // Inject subchannel call tracker to record call completion. complete_pick->subchannel_call_tracker = std::make_unique( @@ -743,36 +779,39 @@ RefCountedPtr XdsClusterImplLb::Helper::CreateSubchannel( const grpc_resolved_address& address, const ChannelArgs& per_address_args, const ChannelArgs& args) { if (parent()->shutting_down_) return nullptr; - // If load reporting is enabled, wrap the subchannel such that it - // includes the locality stats object, which will be used by the Picker. + // Wrap the subchannel so that we pass along the locality labels and + // (if load reporting is enabled) the locality stats object, which + // will be used by the picker. + auto locality_name = per_address_args.GetObjectRef(); + RefCountedPtr locality_stats; if (parent()->cluster_resource_->lrs_load_reporting_server.has_value()) { - auto locality_name = per_address_args.GetObjectRef(); - RefCountedPtr locality_stats = - parent()->xds_client_->AddClusterLocalityStats( - parent()->cluster_resource_->lrs_load_reporting_server.value(), - parent()->config_->cluster_name(), - GetEdsResourceName(*parent()->cluster_resource_), - std::move(locality_name)); - if (locality_stats != nullptr) { - return MakeRefCounted( - parent()->channel_control_helper()->CreateSubchannel( - address, per_address_args, args), - std::move(locality_stats)); + locality_stats = parent()->xds_client_->AddClusterLocalityStats( + parent()->cluster_resource_->lrs_load_reporting_server.value(), + parent()->config_->cluster_name(), + GetEdsResourceName(*parent()->cluster_resource_), locality_name); + if (locality_stats == nullptr) { + gpr_log(GPR_ERROR, + "[xds_cluster_impl_lb %p] Failed to get locality stats object " + "for LRS server %s, cluster %s, EDS service name %s; load " + "reports will not be generated", + parent(), + parent() + ->cluster_resource_->lrs_load_reporting_server->server_uri() + .c_str(), + parent()->config_->cluster_name().c_str(), + GetEdsResourceName(*parent()->cluster_resource_).c_str()); } - gpr_log(GPR_ERROR, - "[xds_cluster_impl_lb %p] Failed to get locality stats object for " - "LRS server %s, cluster %s, EDS service name %s; load reports will " - "not be generated (not wrapping subchannel)", - parent(), - parent() - ->cluster_resource_->lrs_load_reporting_server->server_uri() - .c_str(), - parent()->config_->cluster_name().c_str(), - GetEdsResourceName(*parent()->cluster_resource_).c_str()); - } - // Load reporting not enabled, so don't wrap the subchannel. - return parent()->channel_control_helper()->CreateSubchannel( - address, per_address_args, args); + } + StatsSubchannelWrapper::LocalityData locality_data; + if (locality_stats != nullptr) { + locality_data = std::move(locality_stats); + } else { + locality_data = locality_name->locality_labels(); + } + return MakeRefCounted( + parent()->channel_control_helper()->CreateSubchannel( + address, per_address_args, args), + std::move(locality_data)); } void XdsClusterImplLb::Helper::UpdateState( diff --git a/test/cpp/end2end/xds/xds_cluster_end2end_test.cc b/test/cpp/end2end/xds/xds_cluster_end2end_test.cc index 18562fee895..2bef38e84b8 100644 --- a/test/cpp/end2end/xds/xds_cluster_end2end_test.cc +++ b/test/cpp/end2end/xds/xds_cluster_end2end_test.cc @@ -309,12 +309,13 @@ TEST_P(CdsTest, ClusterChangeAfterAdsCallFails) { WaitForBackend(DEBUG_LOCATION, 1); } -TEST_P(CdsTest, VerifyCsmServiceLabelsParsing) { +TEST_P(CdsTest, MetricLabels) { // Injects a fake client call tracer factory. Try keep this at top. grpc_core::FakeClientCallTracerFactory fake_client_call_tracer_factory; - CreateAndStartBackends(1); + CreateAndStartBackends(2); // Populates EDS resources. - EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}}); + EdsResourceArgs args({{"locality0", CreateEndpointsForBackends(0, 1)}, + {"locality1", CreateEndpointsForBackends(1, 2)}}); balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); // Populates service labels to CDS resources. auto cluster = default_cluster_; @@ -328,17 +329,46 @@ TEST_P(CdsTest, VerifyCsmServiceLabelsParsing) { channel_args.SetPointer(GRPC_ARG_INJECT_FAKE_CLIENT_CALL_TRACER_FACTORY, &fake_client_call_tracer_factory); ResetStub(/*failover_timeout_ms=*/0, &channel_args); - // Sends an RPC and verifies that the service labels are recorded in the fake - // client call tracer. - CheckRpcSendOk(DEBUG_LOCATION); - EXPECT_THAT(fake_client_call_tracer_factory.GetLastFakeClientCallTracer() - ->GetLastCallAttemptTracer() - ->GetOptionalLabels(), - ::testing::ElementsAre(::testing::Pair( - OptionalLabelComponent::kXdsServiceLabels, - ::testing::Pointee(::testing::ElementsAre( - ::testing::Pair("service_name", "myservice"), - ::testing::Pair("service_namespace", "mynamespace")))))); + // Send an RPC to backend 0. + WaitForBackend(DEBUG_LOCATION, 0); + // Verify that the optional labels are recorded in the call tracer. + EXPECT_THAT( + fake_client_call_tracer_factory.GetLastFakeClientCallTracer() + ->GetLastCallAttemptTracer() + ->GetOptionalLabels(), + ::testing::ElementsAre( + ::testing::Pair( + OptionalLabelComponent::kXdsServiceLabels, + ::testing::Pointee(::testing::ElementsAre( + ::testing::Pair("service_name", "myservice"), + ::testing::Pair("service_namespace", "mynamespace")))), + ::testing::Pair( + OptionalLabelComponent::kXdsLocalityLabels, + ::testing::Pointee(::testing::ElementsAre(::testing::Pair( + "grpc.lb.locality", LocalityNameString("locality0"))))))); + // Send an RPC to backend 1. + WaitForBackend(DEBUG_LOCATION, 1); + // Verify that the optional labels are recorded in the call tracer. + EXPECT_THAT( + fake_client_call_tracer_factory.GetLastFakeClientCallTracer() + ->GetLastCallAttemptTracer() + ->GetOptionalLabels(), + ::testing::ElementsAre( + ::testing::Pair( + OptionalLabelComponent::kXdsServiceLabels, + ::testing::Pointee(::testing::ElementsAre( + ::testing::Pair("service_name", "myservice"), + ::testing::Pair("service_namespace", "mynamespace")))), + ::testing::Pair( + OptionalLabelComponent::kXdsLocalityLabels, + ::testing::Pointee(::testing::ElementsAre(::testing::Pair( + "grpc.lb.locality", LocalityNameString("locality1"))))))); + // TODO(yashkt, yijiem): This shutdown shouldn't actually be necessary. + // The only reason it's here is to add a delay before + // fake_client_call_tracer_factory goes out of scope, since there may + // be lingering callbacks in the call stack that are using the + // CallAttemptTracer even after we get here, which would then cause a + // crash. Find a cleaner way to fix this. balancer_->Shutdown(); } diff --git a/test/cpp/end2end/xds/xds_utils.cc b/test/cpp/end2end/xds/xds_utils.cc index 28f58988169..b1d42bdde79 100644 --- a/test/cpp/end2end/xds/xds_utils.cc +++ b/test/cpp/end2end/xds/xds_utils.cc @@ -317,6 +317,12 @@ void XdsResourceUtils::SetRouteConfiguration( } } +std::string XdsResourceUtils::LocalityNameString(absl::string_view sub_zone) { + return absl::StrFormat("{region=\"%s\", zone=\"%s\", sub_zone=\"%s\"}", + kDefaultLocalityRegion, kDefaultLocalityZone, + sub_zone); +} + ClusterLoadAssignment XdsResourceUtils::BuildEdsResource( const EdsResourceArgs& args, absl::string_view eds_service_name) { ClusterLoadAssignment assignment; diff --git a/test/cpp/end2end/xds/xds_utils.h b/test/cpp/end2end/xds/xds_utils.h index 6acfbc6cbe6..e12d36c2cfb 100644 --- a/test/cpp/end2end/xds/xds_utils.h +++ b/test/cpp/end2end/xds/xds_utils.h @@ -195,6 +195,9 @@ class XdsResourceUtils { bool use_rds = false, const Listener* listener_to_copy = nullptr); + // Returns a string representing the locality with the specified sub_zone. + static std::string LocalityNameString(absl::string_view sub_zone); + // Arguments for constructing an EDS resource. struct EdsResourceArgs { // An individual endpoint for a backend running on a specified port. diff --git a/test/cpp/end2end/xds/xds_wrr_end2end_test.cc b/test/cpp/end2end/xds/xds_wrr_end2end_test.cc index 16799140b8a..a89a3021e73 100644 --- a/test/cpp/end2end/xds/xds_wrr_end2end_test.cc +++ b/test/cpp/end2end/xds/xds_wrr_end2end_test.cc @@ -48,12 +48,6 @@ class WrrTest : public XdsEnd2endTest { void SetUp() override { // No-op -- tests must explicitly call InitClient(). } - - static std::string LocalityNameString(absl::string_view sub_zone) { - return absl::StrFormat("{region=\"%s\", zone=\"%s\", sub_zone=\"%s\"}", - kDefaultLocalityRegion, kDefaultLocalityZone, - sub_zone); - } }; INSTANTIATE_TEST_SUITE_P(XdsTest, WrrTest, ::testing::Values(XdsTestType()), From 6f6f0df2baeabfb7563da6fffcac6bcd0b04d149 Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Tue, 5 Mar 2024 12:36:46 -0800 Subject: [PATCH 07/52] Make TestServiceSignaller more generically useful There are tests in another change I'm working on for which we need to do things like: 1) waiting for N RPCs to concurrently pile up on a server 2) doing 1) multiple times sequentially on the same server PiperOrigin-RevId: 612934305 --- test/cpp/end2end/end2end_test.cc | 4 +-- test/cpp/end2end/test_service_impl.h | 49 ++++++++++++++++++++++------ 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/test/cpp/end2end/end2end_test.cc b/test/cpp/end2end/end2end_test.cc index 673ce88315a..6647c331fd9 100644 --- a/test/cpp/end2end/end2end_test.cc +++ b/test/cpp/end2end/end2end_test.cc @@ -1206,9 +1206,9 @@ TEST_P(End2endTest, CancelRpcAfterStart) { s = stub_->Echo(&context, request, &response); }); if (!GetParam().callback_server()) { - service_.ClientWaitUntilRpcStarted(); + EXPECT_EQ(service_.ClientWaitUntilNRpcsStarted(1), 1); } else { - callback_service_.ClientWaitUntilRpcStarted(); + EXPECT_EQ(callback_service_.ClientWaitUntilNRpcsStarted(1), 1); } context.TryCancel(); diff --git a/test/cpp/end2end/test_service_impl.h b/test/cpp/end2end/test_service_impl.h index c6c9edd6541..9415b88c199 100644 --- a/test/cpp/end2end/test_service_impl.h +++ b/test/cpp/end2end/test_service_impl.h @@ -89,11 +89,27 @@ void ServerTryCancel(ServerContext* context); class TestServiceSignaller { public: - void ClientWaitUntilRpcStarted() { - gpr_log(GPR_DEBUG, "*** enter ClientWaitUntilRpcStarted ***"); + // Waits for at least *desired_rpcs* to to be waiting for a server + // continue notification. + // Returns when *desired_rpcs* reaches that amount, or when we've + // surpassed the timeout, whichever happens first. The return value + // is whatever the number of RPCs waiting for server notification is + // at that time. + int ClientWaitUntilNRpcsStarted(int desired_rpcs, absl::Duration timeout) { + gpr_log(GPR_DEBUG, "*** enter ClientWaitUntilNRpcsStarted ***"); + absl::Time deadline = absl::Now() + timeout; + std::chrono::system_clock::time_point chrono_deadline = + absl::ToChronoTime(deadline); std::unique_lock lock(mu_); - cv_rpc_started_.wait(lock, [this] { return rpc_started_; }); - gpr_log(GPR_DEBUG, "*** leave ClientWaitUntilRpcStarted ***"); + cv_rpc_started_.wait_until(lock, chrono_deadline, [this, desired_rpcs] { + gpr_log( + GPR_DEBUG, + "*** desired_rpcs: %d rpcs_waiting_for_server_to_continue_: %d ***", + desired_rpcs, rpcs_waiting_for_server_to_continue_); + return rpcs_waiting_for_server_to_continue_ >= desired_rpcs; + }); + gpr_log(GPR_DEBUG, "*** leave ClientWaitUntilNRpcsStarted ***"); + return rpcs_waiting_for_server_to_continue_; } void ServerWaitToContinue() { gpr_log(GPR_DEBUG, "*** enter ServerWaitToContinue ***"); @@ -104,20 +120,25 @@ class TestServiceSignaller { void SignalClientThatRpcStarted() { gpr_log(GPR_DEBUG, "*** SignalClientThatRpcStarted ***"); std::unique_lock lock(mu_); - rpc_started_ = true; - cv_rpc_started_.notify_one(); + ++rpcs_waiting_for_server_to_continue_; + cv_rpc_started_.notify_all(); } void SignalServerToContinue() { gpr_log(GPR_DEBUG, "*** SignalServerToContinue ***"); std::unique_lock lock(mu_); server_should_continue_ = true; - cv_server_continue_.notify_one(); + cv_server_continue_.notify_all(); + } + void Reset() { + std::unique_lock lock(mu_); + rpcs_waiting_for_server_to_continue_ = 0; + server_should_continue_ = false; } private: std::mutex mu_; std::condition_variable cv_rpc_started_; - bool rpc_started_ /* GUARDED_BY(mu_) */ = false; + int rpcs_waiting_for_server_to_continue_ /* GUARDED_BY(mu_) */ = 0; std::condition_variable cv_server_continue_; bool server_should_continue_ /* GUARDED_BY(mu_) */ = false; }; @@ -451,8 +472,12 @@ class TestMultipleServiceImpl : public RpcService { std::unique_lock lock(mu_); return signal_client_; } - void ClientWaitUntilRpcStarted() { signaller_.ClientWaitUntilRpcStarted(); } + int ClientWaitUntilNRpcsStarted(int desired_rpcs, + absl::Duration timeout = absl::Minutes(1)) { + return signaller_.ClientWaitUntilNRpcsStarted(desired_rpcs, timeout); + } void SignalServerToContinue() { signaller_.SignalServerToContinue(); } + void ResetSignaller() { signaller_.Reset(); } uint64_t RpcsWaitingForClientCancel() { std::unique_lock lock(mu_); return rpcs_waiting_for_client_cancel_; @@ -495,8 +520,12 @@ class CallbackTestServiceImpl std::unique_lock lock(mu_); return signal_client_; } - void ClientWaitUntilRpcStarted() { signaller_.ClientWaitUntilRpcStarted(); } + int ClientWaitUntilNRpcsStarted(int desired_rpcs, + absl::Duration timeout = absl::Minutes(1)) { + return signaller_.ClientWaitUntilNRpcsStarted(desired_rpcs, timeout); + } void SignalServerToContinue() { signaller_.SignalServerToContinue(); } + void ResetSignaller() { signaller_.Reset(); } private: bool signal_client_; From 52fc023bae9f6534c79329b93656a2ef98b5b18c Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Tue, 5 Mar 2024 13:05:02 -0800 Subject: [PATCH 08/52] [CI] Upgraded cocoapods to 1.12.0 (#36053) This is needed to accommodate the recent Protobuf v26 change requiring cocoapod 1.12 or later. Closes #36053 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/36053 from veblush:cocoapod-1.12 ab94e124a6d0f202dc76e735003727548493d688 PiperOrigin-RevId: 612942897 --- tools/internal_ci/helper_scripts/prepare_build_macos_rc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc index b5768a0c317..54a999f149c 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc @@ -99,7 +99,7 @@ then # cocoapods export LANG=en_US.UTF-8 # use "sudo" to avoid permission error on kokoro monterey image - time sudo gem install cocoapods --version 1.11.3 --no-document --user-install + time sudo gem install cocoapods --version 1.12.0 --no-document --user-install # pre-fetch cocoapods master repo's most recent commit only mkdir -p ~/.cocoapods/repos time git clone --depth 1 https://github.com/CocoaPods/Specs.git ~/.cocoapods/repos/master From c332854964a4a71e4499a4c7c35e3e8736072aa9 Mon Sep 17 00:00:00 2001 From: dmaclach Date: Tue, 5 Mar 2024 14:15:02 -0800 Subject: [PATCH 09/52] Use clock_gettime on Apple platforms (#35923) Apple platforms have supported clock_gettime for several years now (since 2016). Simplify implementation and remove required api call to `mach_absolute_time`. Closes #35923 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/35923 from dmaclach:patch-4 45f670d10e056480f3de7e0805f017cdcef6c947 PiperOrigin-RevId: 612965724 --- src/core/lib/gpr/posix/time.cc | 60 ---------------------------------- 1 file changed, 60 deletions(-) diff --git a/src/core/lib/gpr/posix/time.cc b/src/core/lib/gpr/posix/time.cc index 843d2a727d8..050cbf87dbc 100644 --- a/src/core/lib/gpr/posix/time.cc +++ b/src/core/lib/gpr/posix/time.cc @@ -45,7 +45,6 @@ static struct timespec timespec_from_gpr(gpr_timespec gts) { return rv; } -#if _POSIX_TIMERS > 0 || defined(__OpenBSD__) static gpr_timespec gpr_from_timespec(struct timespec ts, gpr_clock_type clock_type) { // @@ -83,65 +82,6 @@ static gpr_timespec now_impl(gpr_clock_type clock_type) { return gpr_from_timespec(now, clock_type); } } -#else -// For some reason Apple's OSes haven't implemented clock_gettime. - -#include -#include -#include - -static double g_time_scale = []() { - mach_timebase_info_data_t tb = {0, 1}; - mach_timebase_info(&tb); - return static_cast(tb.numer) / static_cast(tb.denom); -}(); -static uint64_t g_time_start = mach_absolute_time(); - -void gpr_time_init(void) { gpr_precise_clock_init(); } - -static gpr_timespec now_impl(gpr_clock_type clock) { - gpr_timespec now; - struct timeval now_tv; - double now_dbl; - - now.clock_type = clock; - switch (clock) { - case GPR_CLOCK_REALTIME: - // gettimeofday(...) function may return with a value whose tv_usec is - // greater than 1e6 on iOS The case is resolved with the guard at end of - // this function. - gettimeofday(&now_tv, nullptr); - now.tv_sec = now_tv.tv_sec; - now.tv_nsec = now_tv.tv_usec * 1000; - break; - case GPR_CLOCK_MONOTONIC: - // Add 5 seconds arbitrarily: avoids weird conditions in gprpp/time.cc - // when there's a small number of seconds returned. - now_dbl = 5.0e9 + - ((double)(mach_absolute_time() - g_time_start)) * g_time_scale; - now.tv_sec = (int64_t)(now_dbl * 1e-9); - now.tv_nsec = (int32_t)(now_dbl - ((double)now.tv_sec) * 1e9); - break; - case GPR_CLOCK_PRECISE: - gpr_precise_clock_now(&now); - break; - case GPR_TIMESPAN: - abort(); - } - - // Guard the tv_nsec field in valid range for all clock types - while (GPR_UNLIKELY(now.tv_nsec >= 1e9)) { - now.tv_sec++; - now.tv_nsec -= 1e9; - } - while (GPR_UNLIKELY(now.tv_nsec < 0)) { - now.tv_sec--; - now.tv_nsec += 1e9; - } - - return now; -} -#endif gpr_timespec (*gpr_now_impl)(gpr_clock_type clock_type) = now_impl; From 25ef96c4ad4c94fb76f4cf8ec2afcb39d0183943 Mon Sep 17 00:00:00 2001 From: AJ Heller Date: Tue, 5 Mar 2024 14:44:52 -0800 Subject: [PATCH 10/52] [test] Enable callback benchmarks (#36055) Closes #36055 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/36055 from drfloob:enable-callback-benchmarks 42283a52c9d99605e606e697386289d8707bda11 PiperOrigin-RevId: 612975697 --- test/cpp/qps/client_callback.cc | 8 ++++---- test/cpp/qps/json_run_localhost_scenarios.bzl | 10 ++++++++++ test/cpp/qps/qps_json_driver_scenarios.bzl | 3 +++ tools/run_tests/performance/scenario_config.py | 17 ++++++++++++++--- 4 files changed, 31 insertions(+), 7 deletions(-) diff --git a/test/cpp/qps/client_callback.cc b/test/cpp/qps/client_callback.cc index ff4ffbf07aa..65ad207de63 100644 --- a/test/cpp/qps/client_callback.cc +++ b/test/cpp/qps/client_callback.cc @@ -378,11 +378,11 @@ std::unique_ptr CreateCallbackClient(const ClientConfig& config) { case STREAMING_FROM_CLIENT: case STREAMING_FROM_SERVER: case STREAMING_BOTH_WAYS: - assert(false); - return nullptr; + grpc_core::Crash( + "STREAMING_FROM_* scenarios are not supported by the callback " + "API"); default: - assert(false); - return nullptr; + grpc_core::Crash(absl::StrCat("Unknown RPC type: ", config.rpc_type())); } } diff --git a/test/cpp/qps/json_run_localhost_scenarios.bzl b/test/cpp/qps/json_run_localhost_scenarios.bzl index e7b2820e08c..3eae769161c 100644 --- a/test/cpp/qps/json_run_localhost_scenarios.bzl +++ b/test/cpp/qps/json_run_localhost_scenarios.bzl @@ -29,12 +29,17 @@ JSON_RUN_LOCALHOST_SCENARIOS = { "cpp_protobuf_sync_unary_qps_unconstrained_secure": '\'{"scenarios": [{"name": "cpp_protobuf_sync_unary_qps_unconstrained_secure", "num_servers": 1, "num_clients": 0, "client_config": {"client_type": "SYNC_CLIENT", "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "outstanding_rpcs_per_channel": 1, "client_channels": 16, "async_client_threads": 0, "client_processes": 0, "threads_per_cq": 2, "rpc_type": "UNARY", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "SYNC_SERVER", "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "async_server_threads": 0, "server_processes": 0, "threads_per_cq": 2, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', "cpp_protobuf_async_unary_ping_pong_secure": '\'{"scenarios": [{"name": "cpp_protobuf_async_unary_ping_pong_secure", "num_servers": 1, "num_clients": 1, "client_config": {"client_type": "ASYNC_CLIENT", "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "outstanding_rpcs_per_channel": 1, "client_channels": 1, "async_client_threads": 1, "client_processes": 0, "threads_per_cq": 0, "rpc_type": "UNARY", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "latency"}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "ASYNC_SERVER", "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "async_server_threads": 1, "server_processes": 0, "threads_per_cq": 0, "channel_args": [{"name": "grpc.optimization_target", "str_value": "latency"}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', "cpp_protobuf_async_unary_qps_unconstrained_secure": '\'{"scenarios": [{"name": "cpp_protobuf_async_unary_qps_unconstrained_secure", "num_servers": 1, "num_clients": 0, "client_config": {"client_type": "ASYNC_CLIENT", "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "outstanding_rpcs_per_channel": 100, "client_channels": 16, "async_client_threads": 0, "client_processes": 0, "threads_per_cq": 2, "rpc_type": "UNARY", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "ASYNC_SERVER", "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "async_server_threads": 0, "server_processes": 0, "threads_per_cq": 2, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', + "cpp_protobuf_callback_unary_ping_pong_secure": '\'{"scenarios": [{"name": "cpp_protobuf_callback_unary_ping_pong_secure", "num_servers": 1, "num_clients": 1, "client_config": {"client_type": "CALLBACK_CLIENT", "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "outstanding_rpcs_per_channel": 1, "client_channels": 1, "async_client_threads": 1, "client_processes": 0, "threads_per_cq": 0, "rpc_type": "UNARY", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "latency"}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "CALLBACK_SERVER", "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "async_server_threads": 1, "server_processes": 0, "threads_per_cq": 0, "channel_args": [{"name": "grpc.optimization_target", "str_value": "latency"}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', + "cpp_protobuf_callback_unary_qps_unconstrained_secure": '\'{"scenarios": [{"name": "cpp_protobuf_callback_unary_qps_unconstrained_secure", "num_servers": 1, "num_clients": 0, "client_config": {"client_type": "CALLBACK_CLIENT", "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "outstanding_rpcs_per_channel": 100, "client_channels": 16, "async_client_threads": 0, "client_processes": 0, "threads_per_cq": 2, "rpc_type": "UNARY", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "CALLBACK_SERVER", "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "async_server_threads": 0, "server_processes": 0, "threads_per_cq": 2, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', "cpp_protobuf_sync_streaming_ping_pong_secure": '\'{"scenarios": [{"name": "cpp_protobuf_sync_streaming_ping_pong_secure", "num_servers": 1, "num_clients": 1, "client_config": {"client_type": "SYNC_CLIENT", "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "outstanding_rpcs_per_channel": 1, "client_channels": 1, "async_client_threads": 1, "client_processes": 0, "threads_per_cq": 0, "rpc_type": "STREAMING", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "latency"}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "SYNC_SERVER", "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "async_server_threads": 1, "server_processes": 0, "threads_per_cq": 0, "channel_args": [{"name": "grpc.optimization_target", "str_value": "latency"}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', "cpp_protobuf_sync_streaming_qps_unconstrained_secure": '\'{"scenarios": [{"name": "cpp_protobuf_sync_streaming_qps_unconstrained_secure", "num_servers": 1, "num_clients": 0, "client_config": {"client_type": "SYNC_CLIENT", "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "outstanding_rpcs_per_channel": 1, "client_channels": 16, "async_client_threads": 0, "client_processes": 0, "threads_per_cq": 2, "rpc_type": "STREAMING", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "SYNC_SERVER", "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "async_server_threads": 0, "server_processes": 0, "threads_per_cq": 2, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', "cpp_protobuf_sync_streaming_qps_unconstrained_10mps_secure": '\'{"scenarios": [{"name": "cpp_protobuf_sync_streaming_qps_unconstrained_10mps_secure", "num_servers": 1, "num_clients": 0, "client_config": {"client_type": "SYNC_CLIENT", "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "outstanding_rpcs_per_channel": 1, "client_channels": 16, "async_client_threads": 0, "client_processes": 0, "threads_per_cq": 0, "rpc_type": "STREAMING", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}, "messages_per_stream": 10}, "server_config": {"server_type": "SYNC_SERVER", "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "async_server_threads": 0, "server_processes": 0, "threads_per_cq": 0, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', "cpp_protobuf_async_streaming_ping_pong_secure": '\'{"scenarios": [{"name": "cpp_protobuf_async_streaming_ping_pong_secure", "num_servers": 1, "num_clients": 1, "client_config": {"client_type": "ASYNC_CLIENT", "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "outstanding_rpcs_per_channel": 1, "client_channels": 1, "async_client_threads": 1, "client_processes": 0, "threads_per_cq": 0, "rpc_type": "STREAMING", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "latency"}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "ASYNC_SERVER", "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "async_server_threads": 1, "server_processes": 0, "threads_per_cq": 0, "channel_args": [{"name": "grpc.optimization_target", "str_value": "latency"}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', "cpp_protobuf_async_streaming_qps_unconstrained_secure": '\'{"scenarios": [{"name": "cpp_protobuf_async_streaming_qps_unconstrained_secure", "num_servers": 1, "num_clients": 0, "client_config": {"client_type": "ASYNC_CLIENT", "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "outstanding_rpcs_per_channel": 100, "client_channels": 16, "async_client_threads": 0, "client_processes": 0, "threads_per_cq": 2, "rpc_type": "STREAMING", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "ASYNC_SERVER", "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "async_server_threads": 0, "server_processes": 0, "threads_per_cq": 2, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', "cpp_protobuf_async_streaming_qps_unconstrained_10mps_secure": '\'{"scenarios": [{"name": "cpp_protobuf_async_streaming_qps_unconstrained_10mps_secure", "num_servers": 1, "num_clients": 0, "client_config": {"client_type": "ASYNC_CLIENT", "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "outstanding_rpcs_per_channel": 100, "client_channels": 16, "async_client_threads": 0, "client_processes": 0, "threads_per_cq": 0, "rpc_type": "STREAMING", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}, "messages_per_stream": 10}, "server_config": {"server_type": "ASYNC_SERVER", "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "async_server_threads": 0, "server_processes": 0, "threads_per_cq": 0, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', + "cpp_protobuf_callback_streaming_ping_pong_secure": '\'{"scenarios": [{"name": "cpp_protobuf_callback_streaming_ping_pong_secure", "num_servers": 1, "num_clients": 1, "client_config": {"client_type": "CALLBACK_CLIENT", "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "outstanding_rpcs_per_channel": 1, "client_channels": 1, "async_client_threads": 1, "client_processes": 0, "threads_per_cq": 0, "rpc_type": "STREAMING", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "latency"}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "CALLBACK_SERVER", "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "async_server_threads": 1, "server_processes": 0, "threads_per_cq": 0, "channel_args": [{"name": "grpc.optimization_target", "str_value": "latency"}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', + "cpp_protobuf_callback_streaming_qps_unconstrained_secure": '\'{"scenarios": [{"name": "cpp_protobuf_callback_streaming_qps_unconstrained_secure", "num_servers": 1, "num_clients": 0, "client_config": {"client_type": "CALLBACK_CLIENT", "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "outstanding_rpcs_per_channel": 100, "client_channels": 16, "async_client_threads": 0, "client_processes": 0, "threads_per_cq": 2, "rpc_type": "STREAMING", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "CALLBACK_SERVER", "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "async_server_threads": 0, "server_processes": 0, "threads_per_cq": 2, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', + "cpp_protobuf_callback_streaming_qps_unconstrained_10mps_secure": '\'{"scenarios": [{"name": "cpp_protobuf_callback_streaming_qps_unconstrained_10mps_secure", "num_servers": 1, "num_clients": 0, "client_config": {"client_type": "CALLBACK_CLIENT", "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "outstanding_rpcs_per_channel": 100, "client_channels": 16, "async_client_threads": 0, "client_processes": 0, "threads_per_cq": 0, "rpc_type": "STREAMING", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}, "messages_per_stream": 10}, "server_config": {"server_type": "CALLBACK_SERVER", "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "async_server_threads": 0, "server_processes": 0, "threads_per_cq": 0, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', "cpp_protobuf_sync_streaming_from_client_ping_pong_secure": '\'{"scenarios": [{"name": "cpp_protobuf_sync_streaming_from_client_ping_pong_secure", "num_servers": 1, "num_clients": 1, "client_config": {"client_type": "SYNC_CLIENT", "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "outstanding_rpcs_per_channel": 1, "client_channels": 1, "async_client_threads": 1, "client_processes": 0, "threads_per_cq": 0, "rpc_type": "STREAMING_FROM_CLIENT", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "latency"}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "SYNC_SERVER", "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "async_server_threads": 1, "server_processes": 0, "threads_per_cq": 0, "channel_args": [{"name": "grpc.optimization_target", "str_value": "latency"}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', "cpp_protobuf_sync_streaming_from_client_qps_unconstrained_secure": '\'{"scenarios": [{"name": "cpp_protobuf_sync_streaming_from_client_qps_unconstrained_secure", "num_servers": 1, "num_clients": 0, "client_config": {"client_type": "SYNC_CLIENT", "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "outstanding_rpcs_per_channel": 1, "client_channels": 16, "async_client_threads": 0, "client_processes": 0, "threads_per_cq": 2, "rpc_type": "STREAMING_FROM_CLIENT", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "SYNC_SERVER", "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "async_server_threads": 0, "server_processes": 0, "threads_per_cq": 2, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', "cpp_protobuf_async_streaming_from_client_ping_pong_secure": '\'{"scenarios": [{"name": "cpp_protobuf_async_streaming_from_client_ping_pong_secure", "num_servers": 1, "num_clients": 1, "client_config": {"client_type": "ASYNC_CLIENT", "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "outstanding_rpcs_per_channel": 1, "client_channels": 1, "async_client_threads": 1, "client_processes": 0, "threads_per_cq": 0, "rpc_type": "STREAMING_FROM_CLIENT", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "latency"}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "ASYNC_SERVER", "security_params": {"use_test_ca": true, "server_host_override": "foo.test.google.fr"}, "async_server_threads": 1, "server_processes": 0, "threads_per_cq": 0, "channel_args": [{"name": "grpc.optimization_target", "str_value": "latency"}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', @@ -56,12 +61,17 @@ JSON_RUN_LOCALHOST_SCENARIOS = { "cpp_protobuf_sync_unary_qps_unconstrained_insecure": '\'{"scenarios": [{"name": "cpp_protobuf_sync_unary_qps_unconstrained_insecure", "num_servers": 1, "num_clients": 0, "client_config": {"client_type": "SYNC_CLIENT", "security_params": null, "outstanding_rpcs_per_channel": 1, "client_channels": 16, "async_client_threads": 0, "client_processes": 0, "threads_per_cq": 2, "rpc_type": "UNARY", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "SYNC_SERVER", "security_params": null, "async_server_threads": 0, "server_processes": 0, "threads_per_cq": 2, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', "cpp_protobuf_async_unary_ping_pong_insecure": '\'{"scenarios": [{"name": "cpp_protobuf_async_unary_ping_pong_insecure", "num_servers": 1, "num_clients": 1, "client_config": {"client_type": "ASYNC_CLIENT", "security_params": null, "outstanding_rpcs_per_channel": 1, "client_channels": 1, "async_client_threads": 1, "client_processes": 0, "threads_per_cq": 0, "rpc_type": "UNARY", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "latency"}, {"name": "grpc.minimal_stack", "int_value": 1}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "ASYNC_SERVER", "security_params": null, "async_server_threads": 1, "server_processes": 0, "threads_per_cq": 0, "channel_args": [{"name": "grpc.optimization_target", "str_value": "latency"}, {"name": "grpc.minimal_stack", "int_value": 1}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', "cpp_protobuf_async_unary_qps_unconstrained_insecure": '\'{"scenarios": [{"name": "cpp_protobuf_async_unary_qps_unconstrained_insecure", "num_servers": 1, "num_clients": 0, "client_config": {"client_type": "ASYNC_CLIENT", "security_params": null, "outstanding_rpcs_per_channel": 100, "client_channels": 16, "async_client_threads": 0, "client_processes": 0, "threads_per_cq": 2, "rpc_type": "UNARY", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "ASYNC_SERVER", "security_params": null, "async_server_threads": 0, "server_processes": 0, "threads_per_cq": 2, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', + "cpp_protobuf_callback_unary_ping_pong_insecure": '\'{"scenarios": [{"name": "cpp_protobuf_callback_unary_ping_pong_insecure", "num_servers": 1, "num_clients": 1, "client_config": {"client_type": "CALLBACK_CLIENT", "security_params": null, "outstanding_rpcs_per_channel": 1, "client_channels": 1, "async_client_threads": 1, "client_processes": 0, "threads_per_cq": 0, "rpc_type": "UNARY", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "latency"}, {"name": "grpc.minimal_stack", "int_value": 1}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "CALLBACK_SERVER", "security_params": null, "async_server_threads": 1, "server_processes": 0, "threads_per_cq": 0, "channel_args": [{"name": "grpc.optimization_target", "str_value": "latency"}, {"name": "grpc.minimal_stack", "int_value": 1}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', + "cpp_protobuf_callback_unary_qps_unconstrained_insecure": '\'{"scenarios": [{"name": "cpp_protobuf_callback_unary_qps_unconstrained_insecure", "num_servers": 1, "num_clients": 0, "client_config": {"client_type": "CALLBACK_CLIENT", "security_params": null, "outstanding_rpcs_per_channel": 100, "client_channels": 16, "async_client_threads": 0, "client_processes": 0, "threads_per_cq": 2, "rpc_type": "UNARY", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "CALLBACK_SERVER", "security_params": null, "async_server_threads": 0, "server_processes": 0, "threads_per_cq": 2, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', "cpp_protobuf_sync_streaming_ping_pong_insecure": '\'{"scenarios": [{"name": "cpp_protobuf_sync_streaming_ping_pong_insecure", "num_servers": 1, "num_clients": 1, "client_config": {"client_type": "SYNC_CLIENT", "security_params": null, "outstanding_rpcs_per_channel": 1, "client_channels": 1, "async_client_threads": 1, "client_processes": 0, "threads_per_cq": 0, "rpc_type": "STREAMING", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "latency"}, {"name": "grpc.minimal_stack", "int_value": 1}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "SYNC_SERVER", "security_params": null, "async_server_threads": 1, "server_processes": 0, "threads_per_cq": 0, "channel_args": [{"name": "grpc.optimization_target", "str_value": "latency"}, {"name": "grpc.minimal_stack", "int_value": 1}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', "cpp_protobuf_sync_streaming_qps_unconstrained_insecure": '\'{"scenarios": [{"name": "cpp_protobuf_sync_streaming_qps_unconstrained_insecure", "num_servers": 1, "num_clients": 0, "client_config": {"client_type": "SYNC_CLIENT", "security_params": null, "outstanding_rpcs_per_channel": 1, "client_channels": 16, "async_client_threads": 0, "client_processes": 0, "threads_per_cq": 2, "rpc_type": "STREAMING", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "SYNC_SERVER", "security_params": null, "async_server_threads": 0, "server_processes": 0, "threads_per_cq": 2, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', "cpp_protobuf_sync_streaming_qps_unconstrained_10mps_insecure": '\'{"scenarios": [{"name": "cpp_protobuf_sync_streaming_qps_unconstrained_10mps_insecure", "num_servers": 1, "num_clients": 0, "client_config": {"client_type": "SYNC_CLIENT", "security_params": null, "outstanding_rpcs_per_channel": 1, "client_channels": 16, "async_client_threads": 0, "client_processes": 0, "threads_per_cq": 0, "rpc_type": "STREAMING", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}, "messages_per_stream": 10}, "server_config": {"server_type": "SYNC_SERVER", "security_params": null, "async_server_threads": 0, "server_processes": 0, "threads_per_cq": 0, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', "cpp_protobuf_async_streaming_ping_pong_insecure": '\'{"scenarios": [{"name": "cpp_protobuf_async_streaming_ping_pong_insecure", "num_servers": 1, "num_clients": 1, "client_config": {"client_type": "ASYNC_CLIENT", "security_params": null, "outstanding_rpcs_per_channel": 1, "client_channels": 1, "async_client_threads": 1, "client_processes": 0, "threads_per_cq": 0, "rpc_type": "STREAMING", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "latency"}, {"name": "grpc.minimal_stack", "int_value": 1}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "ASYNC_SERVER", "security_params": null, "async_server_threads": 1, "server_processes": 0, "threads_per_cq": 0, "channel_args": [{"name": "grpc.optimization_target", "str_value": "latency"}, {"name": "grpc.minimal_stack", "int_value": 1}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', "cpp_protobuf_async_streaming_qps_unconstrained_insecure": '\'{"scenarios": [{"name": "cpp_protobuf_async_streaming_qps_unconstrained_insecure", "num_servers": 1, "num_clients": 0, "client_config": {"client_type": "ASYNC_CLIENT", "security_params": null, "outstanding_rpcs_per_channel": 100, "client_channels": 16, "async_client_threads": 0, "client_processes": 0, "threads_per_cq": 2, "rpc_type": "STREAMING", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "ASYNC_SERVER", "security_params": null, "async_server_threads": 0, "server_processes": 0, "threads_per_cq": 2, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', "cpp_protobuf_async_streaming_qps_unconstrained_10mps_insecure": '\'{"scenarios": [{"name": "cpp_protobuf_async_streaming_qps_unconstrained_10mps_insecure", "num_servers": 1, "num_clients": 0, "client_config": {"client_type": "ASYNC_CLIENT", "security_params": null, "outstanding_rpcs_per_channel": 100, "client_channels": 16, "async_client_threads": 0, "client_processes": 0, "threads_per_cq": 0, "rpc_type": "STREAMING", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}, "messages_per_stream": 10}, "server_config": {"server_type": "ASYNC_SERVER", "security_params": null, "async_server_threads": 0, "server_processes": 0, "threads_per_cq": 0, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', + "cpp_protobuf_callback_streaming_ping_pong_insecure": '\'{"scenarios": [{"name": "cpp_protobuf_callback_streaming_ping_pong_insecure", "num_servers": 1, "num_clients": 1, "client_config": {"client_type": "CALLBACK_CLIENT", "security_params": null, "outstanding_rpcs_per_channel": 1, "client_channels": 1, "async_client_threads": 1, "client_processes": 0, "threads_per_cq": 0, "rpc_type": "STREAMING", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "latency"}, {"name": "grpc.minimal_stack", "int_value": 1}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "CALLBACK_SERVER", "security_params": null, "async_server_threads": 1, "server_processes": 0, "threads_per_cq": 0, "channel_args": [{"name": "grpc.optimization_target", "str_value": "latency"}, {"name": "grpc.minimal_stack", "int_value": 1}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', + "cpp_protobuf_callback_streaming_qps_unconstrained_insecure": '\'{"scenarios": [{"name": "cpp_protobuf_callback_streaming_qps_unconstrained_insecure", "num_servers": 1, "num_clients": 0, "client_config": {"client_type": "CALLBACK_CLIENT", "security_params": null, "outstanding_rpcs_per_channel": 100, "client_channels": 16, "async_client_threads": 0, "client_processes": 0, "threads_per_cq": 2, "rpc_type": "STREAMING", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "CALLBACK_SERVER", "security_params": null, "async_server_threads": 0, "server_processes": 0, "threads_per_cq": 2, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', + "cpp_protobuf_callback_streaming_qps_unconstrained_10mps_insecure": '\'{"scenarios": [{"name": "cpp_protobuf_callback_streaming_qps_unconstrained_10mps_insecure", "num_servers": 1, "num_clients": 0, "client_config": {"client_type": "CALLBACK_CLIENT", "security_params": null, "outstanding_rpcs_per_channel": 100, "client_channels": 16, "async_client_threads": 0, "client_processes": 0, "threads_per_cq": 0, "rpc_type": "STREAMING", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}, "messages_per_stream": 10}, "server_config": {"server_type": "CALLBACK_SERVER", "security_params": null, "async_server_threads": 0, "server_processes": 0, "threads_per_cq": 0, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', "cpp_protobuf_sync_streaming_from_client_ping_pong_insecure": '\'{"scenarios": [{"name": "cpp_protobuf_sync_streaming_from_client_ping_pong_insecure", "num_servers": 1, "num_clients": 1, "client_config": {"client_type": "SYNC_CLIENT", "security_params": null, "outstanding_rpcs_per_channel": 1, "client_channels": 1, "async_client_threads": 1, "client_processes": 0, "threads_per_cq": 0, "rpc_type": "STREAMING_FROM_CLIENT", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "latency"}, {"name": "grpc.minimal_stack", "int_value": 1}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "SYNC_SERVER", "security_params": null, "async_server_threads": 1, "server_processes": 0, "threads_per_cq": 0, "channel_args": [{"name": "grpc.optimization_target", "str_value": "latency"}, {"name": "grpc.minimal_stack", "int_value": 1}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', "cpp_protobuf_sync_streaming_from_client_qps_unconstrained_insecure": '\'{"scenarios": [{"name": "cpp_protobuf_sync_streaming_from_client_qps_unconstrained_insecure", "num_servers": 1, "num_clients": 0, "client_config": {"client_type": "SYNC_CLIENT", "security_params": null, "outstanding_rpcs_per_channel": 1, "client_channels": 16, "async_client_threads": 0, "client_processes": 0, "threads_per_cq": 2, "rpc_type": "STREAMING_FROM_CLIENT", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "SYNC_SERVER", "security_params": null, "async_server_threads": 0, "server_processes": 0, "threads_per_cq": 2, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', "cpp_protobuf_async_streaming_from_client_ping_pong_insecure": '\'{"scenarios": [{"name": "cpp_protobuf_async_streaming_from_client_ping_pong_insecure", "num_servers": 1, "num_clients": 1, "client_config": {"client_type": "ASYNC_CLIENT", "security_params": null, "outstanding_rpcs_per_channel": 1, "client_channels": 1, "async_client_threads": 1, "client_processes": 0, "threads_per_cq": 0, "rpc_type": "STREAMING_FROM_CLIENT", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "latency"}, {"name": "grpc.minimal_stack", "int_value": 1}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "ASYNC_SERVER", "security_params": null, "async_server_threads": 1, "server_processes": 0, "threads_per_cq": 0, "channel_args": [{"name": "grpc.optimization_target", "str_value": "latency"}, {"name": "grpc.minimal_stack", "int_value": 1}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', diff --git a/test/cpp/qps/qps_json_driver_scenarios.bzl b/test/cpp/qps/qps_json_driver_scenarios.bzl index fe3a63784ce..82c01076c3c 100644 --- a/test/cpp/qps/qps_json_driver_scenarios.bzl +++ b/test/cpp/qps/qps_json_driver_scenarios.bzl @@ -26,10 +26,13 @@ QPS_JSON_DRIVER_SCENARIOS = { "cpp_protobuf_async_unary_ping_pong_insecure_1MB": '\'{"scenarios": [{"name": "cpp_protobuf_async_unary_ping_pong_insecure_1MB", "num_servers": 1, "num_clients": 1, "client_config": {"client_type": "ASYNC_CLIENT", "security_params": null, "outstanding_rpcs_per_channel": 1, "client_channels": 1, "async_client_threads": 1, "client_processes": 0, "threads_per_cq": 0, "rpc_type": "UNARY", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "latency"}, {"name": "grpc.minimal_stack", "int_value": 1}], "payload_config": {"simple_params": {"req_size": 1048576, "resp_size": 1048576}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "ASYNC_SERVER", "security_params": null, "async_server_threads": 0, "server_processes": 0, "threads_per_cq": 0, "channel_args": [{"name": "grpc.optimization_target", "str_value": "latency"}, {"name": "grpc.minimal_stack", "int_value": 1}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', "cpp_protobuf_sync_unary_qps_unconstrained_insecure": '\'{"scenarios": [{"name": "cpp_protobuf_sync_unary_qps_unconstrained_insecure", "num_servers": 1, "num_clients": 0, "client_config": {"client_type": "SYNC_CLIENT", "security_params": null, "outstanding_rpcs_per_channel": 1, "client_channels": 16, "async_client_threads": 0, "client_processes": 0, "threads_per_cq": 2, "rpc_type": "UNARY", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "SYNC_SERVER", "security_params": null, "async_server_threads": 0, "server_processes": 0, "threads_per_cq": 2, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', "cpp_protobuf_async_unary_qps_unconstrained_insecure": '\'{"scenarios": [{"name": "cpp_protobuf_async_unary_qps_unconstrained_insecure", "num_servers": 1, "num_clients": 0, "client_config": {"client_type": "ASYNC_CLIENT", "security_params": null, "outstanding_rpcs_per_channel": 100, "client_channels": 16, "async_client_threads": 0, "client_processes": 0, "threads_per_cq": 2, "rpc_type": "UNARY", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "ASYNC_SERVER", "security_params": null, "async_server_threads": 0, "server_processes": 0, "threads_per_cq": 2, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', + "cpp_protobuf_callback_unary_qps_unconstrained_insecure": '\'{"scenarios": [{"name": "cpp_protobuf_callback_unary_qps_unconstrained_insecure", "num_servers": 1, "num_clients": 0, "client_config": {"client_type": "CALLBACK_CLIENT", "security_params": null, "outstanding_rpcs_per_channel": 100, "client_channels": 16, "async_client_threads": 0, "client_processes": 0, "threads_per_cq": 2, "rpc_type": "UNARY", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "CALLBACK_SERVER", "security_params": null, "async_server_threads": 0, "server_processes": 0, "threads_per_cq": 2, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', "cpp_protobuf_sync_streaming_qps_unconstrained_insecure": '\'{"scenarios": [{"name": "cpp_protobuf_sync_streaming_qps_unconstrained_insecure", "num_servers": 1, "num_clients": 0, "client_config": {"client_type": "SYNC_CLIENT", "security_params": null, "outstanding_rpcs_per_channel": 1, "client_channels": 16, "async_client_threads": 0, "client_processes": 0, "threads_per_cq": 2, "rpc_type": "STREAMING", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "SYNC_SERVER", "security_params": null, "async_server_threads": 0, "server_processes": 0, "threads_per_cq": 2, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', "cpp_protobuf_sync_streaming_qps_unconstrained_10mps_insecure": '\'{"scenarios": [{"name": "cpp_protobuf_sync_streaming_qps_unconstrained_10mps_insecure", "num_servers": 1, "num_clients": 0, "client_config": {"client_type": "SYNC_CLIENT", "security_params": null, "outstanding_rpcs_per_channel": 1, "client_channels": 16, "async_client_threads": 0, "client_processes": 0, "threads_per_cq": 0, "rpc_type": "STREAMING", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}, "messages_per_stream": 10}, "server_config": {"server_type": "SYNC_SERVER", "security_params": null, "async_server_threads": 0, "server_processes": 0, "threads_per_cq": 0, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', "cpp_protobuf_async_streaming_qps_unconstrained_insecure": '\'{"scenarios": [{"name": "cpp_protobuf_async_streaming_qps_unconstrained_insecure", "num_servers": 1, "num_clients": 0, "client_config": {"client_type": "ASYNC_CLIENT", "security_params": null, "outstanding_rpcs_per_channel": 100, "client_channels": 16, "async_client_threads": 0, "client_processes": 0, "threads_per_cq": 2, "rpc_type": "STREAMING", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "ASYNC_SERVER", "security_params": null, "async_server_threads": 0, "server_processes": 0, "threads_per_cq": 2, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', "cpp_protobuf_async_streaming_qps_unconstrained_10mps_insecure": '\'{"scenarios": [{"name": "cpp_protobuf_async_streaming_qps_unconstrained_10mps_insecure", "num_servers": 1, "num_clients": 0, "client_config": {"client_type": "ASYNC_CLIENT", "security_params": null, "outstanding_rpcs_per_channel": 100, "client_channels": 16, "async_client_threads": 0, "client_processes": 0, "threads_per_cq": 0, "rpc_type": "STREAMING", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}, "messages_per_stream": 10}, "server_config": {"server_type": "ASYNC_SERVER", "security_params": null, "async_server_threads": 0, "server_processes": 0, "threads_per_cq": 0, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', + "cpp_protobuf_callback_streaming_qps_unconstrained_insecure": '\'{"scenarios": [{"name": "cpp_protobuf_callback_streaming_qps_unconstrained_insecure", "num_servers": 1, "num_clients": 0, "client_config": {"client_type": "CALLBACK_CLIENT", "security_params": null, "outstanding_rpcs_per_channel": 100, "client_channels": 16, "async_client_threads": 0, "client_processes": 0, "threads_per_cq": 2, "rpc_type": "STREAMING", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "CALLBACK_SERVER", "security_params": null, "async_server_threads": 0, "server_processes": 0, "threads_per_cq": 2, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', + "cpp_protobuf_callback_streaming_qps_unconstrained_10mps_insecure": '\'{"scenarios": [{"name": "cpp_protobuf_callback_streaming_qps_unconstrained_10mps_insecure", "num_servers": 1, "num_clients": 0, "client_config": {"client_type": "CALLBACK_CLIENT", "security_params": null, "outstanding_rpcs_per_channel": 100, "client_channels": 16, "async_client_threads": 0, "client_processes": 0, "threads_per_cq": 0, "rpc_type": "STREAMING", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}, "messages_per_stream": 10}, "server_config": {"server_type": "CALLBACK_SERVER", "security_params": null, "async_server_threads": 0, "server_processes": 0, "threads_per_cq": 0, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', "cpp_protobuf_sync_streaming_from_client_qps_unconstrained_insecure": '\'{"scenarios": [{"name": "cpp_protobuf_sync_streaming_from_client_qps_unconstrained_insecure", "num_servers": 1, "num_clients": 0, "client_config": {"client_type": "SYNC_CLIENT", "security_params": null, "outstanding_rpcs_per_channel": 1, "client_channels": 16, "async_client_threads": 0, "client_processes": 0, "threads_per_cq": 2, "rpc_type": "STREAMING_FROM_CLIENT", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "SYNC_SERVER", "security_params": null, "async_server_threads": 0, "server_processes": 0, "threads_per_cq": 2, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', "cpp_protobuf_async_streaming_from_client_qps_unconstrained_insecure": '\'{"scenarios": [{"name": "cpp_protobuf_async_streaming_from_client_qps_unconstrained_insecure", "num_servers": 1, "num_clients": 0, "client_config": {"client_type": "ASYNC_CLIENT", "security_params": null, "outstanding_rpcs_per_channel": 100, "client_channels": 16, "async_client_threads": 0, "client_processes": 0, "threads_per_cq": 2, "rpc_type": "STREAMING_FROM_CLIENT", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "ASYNC_SERVER", "security_params": null, "async_server_threads": 0, "server_processes": 0, "threads_per_cq": 2, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', "cpp_protobuf_sync_streaming_from_server_qps_unconstrained_insecure": '\'{"scenarios": [{"name": "cpp_protobuf_sync_streaming_from_server_qps_unconstrained_insecure", "num_servers": 1, "num_clients": 0, "client_config": {"client_type": "SYNC_CLIENT", "security_params": null, "outstanding_rpcs_per_channel": 1, "client_channels": 4, "async_client_threads": 0, "client_processes": 0, "threads_per_cq": 2, "rpc_type": "STREAMING_FROM_SERVER", "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}], "payload_config": {"simple_params": {"req_size": 0, "resp_size": 0}}, "load_params": {"closed_loop": {}}}, "server_config": {"server_type": "SYNC_SERVER", "security_params": null, "async_server_threads": 0, "server_processes": 0, "threads_per_cq": 2, "channel_args": [{"name": "grpc.optimization_target", "str_value": "throughput"}, {"name": "grpc.minimal_stack", "int_value": 1}]}, "warmup_seconds": 0, "benchmark_seconds": 1}]}\'', diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index 1bbc7618693..0445f6bb920 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -44,7 +44,12 @@ HISTOGRAM_PARAMS = { # target number of RPCs outstanding on across all client channels in # non-ping-pong tests (since we can only specify per-channel numbers, the # actual target will be slightly higher) -OUTSTANDING_REQUESTS = {"async": 6400, "async-limited": 800, "sync": 1000} +OUTSTANDING_REQUESTS = { + "async": 6400, + "async-limited": 800, + "sync": 1000, + "callback": 6400, +} # wide is the number of client channels in multi-channel tests (1 otherwise) WIDE = 64 @@ -561,7 +566,14 @@ class CXXLanguage(Language): maybe_dashboard = ( [DASHBOARD] if rpc_type in ("unary", "streaming") else [] ) - for synchronicity in ["sync", "async"]: + for synchronicity in ["sync", "async", "callback"]: + # TODO(hork): Re-enable when the callback scenarios support + # one-sided streaming. + if synchronicity == "callback" and rpc_type in ( + "streaming_from_client", + "streaming_from_server", + ): + continue yield _ping_pong_scenario( "cpp_protobuf_%s_%s_ping_pong_%s" % (synchronicity, rpc_type, secstr), @@ -574,7 +586,6 @@ class CXXLanguage(Language): categories=list(DEFAULT_CATEGORIES) + maybe_dashboard, warmup_seconds=CXX_WARMUP_SECONDS, ) - for size in geometric_progression( 1, 1024 * 1024 * 1024 + 1, 8 ): From 419a070078a9b4eeae7cf4a61874adba9414eb26 Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Tue, 5 Mar 2024 17:55:02 -0800 Subject: [PATCH 11/52] Allow configuring server_notify_client_when_started field in xds e2e test lib While we're here, also change the Echo service handler to mutate clients before handling the RPC, mainly so that it can be reasoned about in cases involving client cancellation. PiperOrigin-RevId: 613029541 --- test/cpp/end2end/xds/xds_end2end_test_lib.cc | 3 +++ test/cpp/end2end/xds/xds_end2end_test_lib.h | 15 +++++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/test/cpp/end2end/xds/xds_end2end_test_lib.cc b/test/cpp/end2end/xds/xds_end2end_test_lib.cc index c398694266f..6f69e351cfb 100644 --- a/test/cpp/end2end/xds/xds_end2end_test_lib.cc +++ b/test/cpp/end2end/xds/xds_end2end_test_lib.cc @@ -361,6 +361,9 @@ void XdsEnd2endTest::RpcOptions::SetupRpc(ClientContext* context, if (backend_metrics.has_value()) { *request->mutable_param()->mutable_backend_metrics() = *backend_metrics; } + if (server_notify_client_when_started) { + request->mutable_param()->set_server_notify_client_when_started(true); + } } // diff --git a/test/cpp/end2end/xds/xds_end2end_test_lib.h b/test/cpp/end2end/xds/xds_end2end_test_lib.h index 1b9ade11905..86cc867d008 100644 --- a/test/cpp/end2end/xds/xds_end2end_test_lib.h +++ b/test/cpp/end2end/xds/xds_end2end_test_lib.h @@ -299,10 +299,6 @@ class XdsEnd2endTest : public ::testing::TestWithParam, auto peer_identity = context->auth_context()->GetPeerIdentity(); CountedService< TestMultipleServiceImpl>::IncreaseRequestCount(); - const auto status = TestMultipleServiceImpl::Echo( - context, request, response); - CountedService< - TestMultipleServiceImpl>::IncreaseResponseCount(); { grpc_core::MutexLock lock(&mu_); clients_.insert(context->peer()); @@ -322,6 +318,10 @@ class XdsEnd2endTest : public ::testing::TestWithParam, recorder->RecordNamedMetric(key, p.second); } } + const auto status = TestMultipleServiceImpl::Echo( + context, request, response); + CountedService< + TestMultipleServiceImpl>::IncreaseResponseCount(); return status; } @@ -599,6 +599,7 @@ class XdsEnd2endTest : public ::testing::TestWithParam, bool skip_cancelled_check = false; StatusCode server_expected_error = StatusCode::OK; absl::optional backend_metrics; + bool server_notify_client_when_started = false; RpcOptions() {} @@ -664,6 +665,12 @@ class XdsEnd2endTest : public ::testing::TestWithParam, return *this; } + RpcOptions& set_server_notify_client_when_started( + bool rpc_server_notify_client_when_started) { + server_notify_client_when_started = rpc_server_notify_client_when_started; + return *this; + } + // Populates context and request. void SetupRpc(ClientContext* context, EchoRequest* request) const; }; From 1491233a805a790ac5731e0ef5ae8223c591ce0a Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Tue, 5 Mar 2024 17:57:00 -0800 Subject: [PATCH 12/52] Expose LB policy test lib SubchannelState::state_tracker_ to tests PiperOrigin-RevId: 613029941 --- test/core/client_channel/lb_policy/lb_policy_test_lib.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/core/client_channel/lb_policy/lb_policy_test_lib.h b/test/core/client_channel/lb_policy/lb_policy_test_lib.h index 7048fec08d3..b7444add2b6 100644 --- a/test/core/client_channel/lb_policy/lb_policy_test_lib.h +++ b/test/core/client_channel/lb_policy/lb_policy_test_lib.h @@ -402,6 +402,10 @@ class LoadBalancingPolicyTest : public ::testing::Test { return test_->work_serializer_; } + ConnectivityStateTracker& state_tracker() { + return state_tracker_; + } + private: const std::string address_; LoadBalancingPolicyTest* const test_; From 02276f47743b5549a60a5bb18f707d6c0862becc Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Wed, 6 Mar 2024 08:31:48 -0800 Subject: [PATCH 13/52] [Misc] Added third_party/utf8_range to .bazelignore (#36054) This is to clam down bazel which is eager to build everything in the repo. --- .bazelignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.bazelignore b/.bazelignore index 7ace587d0ff..13ca0ec3c7f 100644 --- a/.bazelignore +++ b/.bazelignore @@ -19,6 +19,7 @@ third_party/protobuf third_party/protoc-gen-validate third_party/re2 third_party/upb +third_party/utf8_range third_party/xds test/distrib/bazel/cpp From 7b98cda1f4c4a3836d02b4be4018926f62aaf42b Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Thu, 7 Mar 2024 10:23:45 -0800 Subject: [PATCH 14/52] No public description PiperOrigin-RevId: 613626993 --- Makefile | 2331 +++++------------------------ Rakefile | 1 - build_handwritten.yaml | 7 +- grpc.gyp | 12 + src/php/ext/grpc/config.m4 | 11 - src/ruby/ext/grpc/extconf.rb | 2 - templates/CMakeLists.txt.template | 25 +- templates/Makefile.template | 199 +-- tools/buildgen/_utils.py | 42 +- 9 files changed, 470 insertions(+), 2160 deletions(-) diff --git a/Makefile b/Makefile index 2fd360f7202..4bdbbec686f 100644 --- a/Makefile +++ b/Makefile @@ -268,49 +268,6 @@ ifndef VALID_CONFIG_$(CONFIG) $(error Invalid CONFIG value '$(CONFIG)') endif -ifeq ($(SYSTEM),Linux) -TMPOUT = /dev/null -else -TMPOUT = `mktemp /tmp/test-out-XXXXXX` -endif - -CHECK_NO_CXX14_COMPAT_WORKS_CMD = $(CC) -std=c++14 -Werror -Wno-c++14-compat -o $(TMPOUT) -c test/build/no-c++14-compat.cc -HAS_WORKING_NO_CXX14_COMPAT = $(shell $(CHECK_NO_CXX14_COMPAT_WORKS_CMD) 2> /dev/null && echo true || echo false) -ifeq ($(HAS_WORKING_NO_CXX14_COMPAT),true) -W_NO_CXX14_COMPAT=-Wno-c++14-compat -endif - -CHECK_EXTRA_SEMI_WORKS_CMD = $(CC) -std=c99 -Werror -Wextra-semi -o $(TMPOUT) -c test/build/extra-semi.c -HAS_WORKING_EXTRA_SEMI = $(shell $(CHECK_EXTRA_SEMI_WORKS_CMD) 2> /dev/null && echo true || echo false) -ifeq ($(HAS_WORKING_EXTRA_SEMI),true) -W_EXTRA_SEMI=-Wextra-semi -NO_W_EXTRA_SEMI=-Wno-extra-semi -endif -CHECK_NO_SHIFT_NEGATIVE_VALUE_WORKS_CMD = $(CC) -std=c99 -Werror -Wno-shift-negative-value -o $(TMPOUT) -c test/build/no-shift-negative-value.c -HAS_WORKING_NO_SHIFT_NEGATIVE_VALUE = $(shell $(CHECK_NO_SHIFT_NEGATIVE_VALUE_WORKS_CMD) 2> /dev/null && echo true || echo false) -ifeq ($(HAS_WORKING_NO_SHIFT_NEGATIVE_VALUE),true) -W_NO_SHIFT_NEGATIVE_VALUE=-Wno-shift-negative-value -NO_W_NO_SHIFT_NEGATIVE_VALUE=-Wshift-negative-value -endif -CHECK_NO_UNUSED_BUT_SET_VARIABLE_WORKS_CMD = $(CC) -std=c99 -Werror -Wno-unused-but-set-variable -o $(TMPOUT) -c test/build/no-unused-but-set-variable.c -HAS_WORKING_NO_UNUSED_BUT_SET_VARIABLE = $(shell $(CHECK_NO_UNUSED_BUT_SET_VARIABLE_WORKS_CMD) 2> /dev/null && echo true || echo false) -ifeq ($(HAS_WORKING_NO_UNUSED_BUT_SET_VARIABLE),true) -W_NO_UNUSED_BUT_SET_VARIABLE=-Wno-unused-but-set-variable -NO_W_NO_UNUSED_BUT_SET_VARIABLE=-Wunused-but-set-variable -endif -CHECK_NO_MAYBE_UNINITIALIZED_WORKS_CMD = $(CC) -std=c99 -Werror -Wno-maybe-uninitialized -o $(TMPOUT) -c test/build/no-maybe-uninitialized.c -HAS_WORKING_NO_MAYBE_UNINITIALIZED = $(shell $(CHECK_NO_MAYBE_UNINITIALIZED_WORKS_CMD) 2> /dev/null && echo true || echo false) -ifeq ($(HAS_WORKING_NO_MAYBE_UNINITIALIZED),true) -W_NO_MAYBE_UNINITIALIZED=-Wno-maybe-uninitialized -NO_W_NO_MAYBE_UNINITIALIZED=-Wmaybe-uninitialized -endif -CHECK_NO_UNKNOWN_WARNING_OPTION_WORKS_CMD = $(CC) -std=c99 -Werror -Wno-unknown-warning-option -o $(TMPOUT) -c test/build/no-unknown-warning-option.c -HAS_WORKING_NO_UNKNOWN_WARNING_OPTION = $(shell $(CHECK_NO_UNKNOWN_WARNING_OPTION_WORKS_CMD) 2> /dev/null && echo true || echo false) -ifeq ($(HAS_WORKING_NO_UNKNOWN_WARNING_OPTION),true) -W_NO_UNKNOWN_WARNING_OPTION=-Wno-unknown-warning-option -NO_W_NO_UNKNOWN_WARNING_OPTION=-Wunknown-warning-option -endif - # The HOST compiler settings are used to compile the protoc plugins. # In most cases, you won't have to change anything, but if you are # cross-compiling, you can override these variables from GNU make's @@ -321,14 +278,14 @@ HOST_CXX ?= $(CXX) HOST_LD ?= $(LD) HOST_LDXX ?= $(LDXX) -CFLAGS += -std=c11 $(W_EXTRA_SEMI) +CFLAGS += -std=c11 CXXFLAGS += -std=c++14 ifeq ($(SYSTEM),Darwin) CXXFLAGS += -stdlib=libc++ LDFLAGS += -framework CoreFoundation endif CFLAGS += -g -CPPFLAGS += -g -Wall -Wextra -DOSATOMIC_USE_INLINED=1 -Ithird_party/abseil-cpp -Ithird_party/re2 -Ithird_party/upb -Isrc/core/ext/upb-gen -Isrc/core/ext/upbdefs-gen -Ithird_party/utf8_range -Ithird_party/xxhash +CPPFLAGS += -g -Wall -Wextra -DOSATOMIC_USE_INLINED=1 -Ithird_party/abseil-cpp -Ithird_party/re2 -Ithird_party/upb -Isrc/core/ext/upb-gen -Isrc/core/ext/upbdefs-gen -Ithird_party/utf8_range -Ithird_party/xxhash -Ithird_party/cares/cares/include -Ithird_party/cares -Ithird_party/cares/cares -Ithird_party/address_sorting/include COREFLAGS += -fno-exceptions LDFLAGS += -g @@ -494,36 +451,6 @@ else LIBS += z endif -# Setup c-ares dependency - -ifeq ($(wildcard third_party/cares/cares/include/ares.h),) -HAS_EMBEDDED_CARES = false -else -HAS_EMBEDDED_CARES = true -endif - -ifeq ($(HAS_EMBEDDED_CARES),true) -EMBED_CARES ?= true -else -# only building with c-ares from submodule is supported -DEP_MISSING += cares -EMBED_CARES ?= broken -endif - -ifeq ($(EMBED_CARES),true) -CPPFLAGS := -Ithird_party/cares/cares/include -Ithird_party/cares -Ithird_party/cares/cares $(CPPFLAGS) -endif - -# Setup address_sorting dependency - -# TODO(jtattermusch): should the include be added elsewhere? -CPPFLAGS := -Ithird_party/address_sorting/include $(CPPFLAGS) - -# Setup abseil dependency - -GRPC_ABSEIL_DEP = $(LIBDIR)/$(CONFIG)/libgrpc_abseil.a -GRPC_ABSEIL_MERGE_LIBS = $(LIBDIR)/$(CONFIG)/libgrpc_abseil.a - # Setup boringssl dependency ifeq ($(wildcard third_party/boringssl-with-bazel/src/include/openssl/ssl.h),) @@ -558,10 +485,9 @@ ifeq ($(MAKECMDGOALS),clean) NO_DEPS = true endif -.SECONDARY = %.pb.h %.pb.cc - ifeq ($(DEP_MISSING),) all: static shared + dep_error: @echo "You shouldn't see this message - all of your dependencies are correct." else @@ -634,24 +560,23 @@ install_not_supported_error: install_not_supported_message stop stop: @false - run_dep_checks: @echo "run_dep_checks target has been deprecated." static: static_c static_cxx -static_c: cache.mk $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a $(LIBDIR)/$(CONFIG)/libre2.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(LIBDIR)/$(CONFIG)/libupb_json_lib.a $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBDIR)/$(CONFIG)/libupb_textformat_lib.a $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a +static_c: cache.mk $(LIBDIR)/$(CONFIG)/libgrpc.a static_cxx: cache.mk shared: shared_c shared_cxx -shared_c: cache.mk $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)address_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)re2$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)upb_base_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)upb_json_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)upb_mem_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)upb_message_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)upb_textformat_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)utf8_range_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) +shared_c: cache.mk $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) shared_cxx: cache.mk privatelibs: privatelibs_c privatelibs_cxx -privatelibs_c: $(LIBDIR)/$(CONFIG)/libz.a $(LIBDIR)/$(CONFIG)/libcares.a +privatelibs_c: $(LIBDIR)/$(CONFIG)/libcares.a $(LIBDIR)/$(CONFIG)/libz.a ifeq ($(EMBED_OPENSSL),true) privatelibs_cxx: else @@ -667,28 +592,8 @@ strip-shared: strip-shared_c strip-shared_cxx strip-static_c: static_c ifeq ($(CONFIG),opt) - $(E) "[STRIP] Stripping libaddress_sorting.a" - $(Q) $(STRIP) $(LIBDIR)/$(CONFIG)/libaddress_sorting.a - $(E) "[STRIP] Stripping libgpr.a" - $(Q) $(STRIP) $(LIBDIR)/$(CONFIG)/libgpr.a $(E) "[STRIP] Stripping libgrpc.a" $(Q) $(STRIP) $(LIBDIR)/$(CONFIG)/libgrpc.a - $(E) "[STRIP] Stripping libgrpc_unsecure.a" - $(Q) $(STRIP) $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a - $(E) "[STRIP] Stripping libre2.a" - $(Q) $(STRIP) $(LIBDIR)/$(CONFIG)/libre2.a - $(E) "[STRIP] Stripping libupb_base_lib.a" - $(Q) $(STRIP) $(LIBDIR)/$(CONFIG)/libupb_base_lib.a - $(E) "[STRIP] Stripping libupb_json_lib.a" - $(Q) $(STRIP) $(LIBDIR)/$(CONFIG)/libupb_json_lib.a - $(E) "[STRIP] Stripping libupb_mem_lib.a" - $(Q) $(STRIP) $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a - $(E) "[STRIP] Stripping libupb_message_lib.a" - $(Q) $(STRIP) $(LIBDIR)/$(CONFIG)/libupb_message_lib.a - $(E) "[STRIP] Stripping libupb_textformat_lib.a" - $(Q) $(STRIP) $(LIBDIR)/$(CONFIG)/libupb_textformat_lib.a - $(E) "[STRIP] Stripping libutf8_range_lib.a" - $(Q) $(STRIP) $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a endif strip-static_cxx: static_cxx @@ -697,28 +602,8 @@ endif strip-shared_c: shared_c ifeq ($(CONFIG),opt) - $(E) "[STRIP] Stripping $(SHARED_PREFIX)address_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE)" - $(Q) $(STRIP) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)address_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) - $(E) "[STRIP] Stripping $(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE)" - $(Q) $(STRIP) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(E) "[STRIP] Stripping $(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE)" $(Q) $(STRIP) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) - $(E) "[STRIP] Stripping $(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE)" - $(Q) $(STRIP) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) - $(E) "[STRIP] Stripping $(SHARED_PREFIX)re2$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE)" - $(Q) $(STRIP) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)re2$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) - $(E) "[STRIP] Stripping $(SHARED_PREFIX)upb_base_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE)" - $(Q) $(STRIP) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)upb_base_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) - $(E) "[STRIP] Stripping $(SHARED_PREFIX)upb_json_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE)" - $(Q) $(STRIP) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)upb_json_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) - $(E) "[STRIP] Stripping $(SHARED_PREFIX)upb_mem_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE)" - $(Q) $(STRIP) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)upb_mem_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) - $(E) "[STRIP] Stripping $(SHARED_PREFIX)upb_message_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE)" - $(Q) $(STRIP) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)upb_message_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) - $(E) "[STRIP] Stripping $(SHARED_PREFIX)upb_textformat_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE)" - $(Q) $(STRIP) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)upb_textformat_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) - $(E) "[STRIP] Stripping $(SHARED_PREFIX)utf8_range_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE)" - $(Q) $(STRIP) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)utf8_range_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) endif strip-shared_cxx: shared_cxx @@ -749,11 +634,6 @@ $(OBJDIR)/$(CONFIG)/src/core/%.o : src/core/%.cc $(Q) mkdir -p `dirname $@` $(Q) $(CXX) $(CPPFLAGS) $(CXXFLAGS) $(COREFLAGS) -MMD -MF $(addsuffix .dep, $(basename $@)) -c -o $@ $< -$(OBJDIR)/$(CONFIG)/test/core/%.o : test/core/%.cc - $(E) "[CXX] Compiling $<" - $(Q) mkdir -p `dirname $@` - $(Q) $(CXX) $(CPPFLAGS) $(CXXFLAGS) $(COREFLAGS) -MMD -MF $(addsuffix .dep, $(basename $@)) -c -o $@ $< - $(OBJDIR)/$(CONFIG)/%.o : %.cc $(E) "[CXX] Compiling $<" $(Q) mkdir -p `dirname $@` @@ -782,181 +662,9 @@ clean: # The various libraries -# start of build recipe for library "address_sorting" (generated by makelib(lib) template function) -# deps: [] -# transitive_deps: [] -LIBADDRESS_SORTING_SRC = \ - third_party/address_sorting/address_sorting.c \ - third_party/address_sorting/address_sorting_posix.c \ - third_party/address_sorting/address_sorting_windows.c \ - -PUBLIC_HEADERS_C += \ - -LIBADDRESS_SORTING_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBADDRESS_SORTING_SRC)))) - - -# static library for "address_sorting" -$(LIBDIR)/$(CONFIG)/libaddress_sorting.a: $(LIBADDRESS_SORTING_OBJS) - $(E) "[AR] Creating $@" - $(Q) mkdir -p `dirname $@` - $(Q) rm -f $(LIBDIR)/$(CONFIG)/libaddress_sorting.a - $(Q) $(AR) $(ARFLAGS) $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBADDRESS_SORTING_OBJS) -ifeq ($(SYSTEM),Darwin) - $(Q) $(RANLIB) $(RANLIBFLAGS) $(LIBDIR)/$(CONFIG)/libaddress_sorting.a -endif - -# shared library for "address_sorting" -ifeq ($(SYSTEM),MINGW32) -$(LIBDIR)/$(CONFIG)/address_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBADDRESS_SORTING_OBJS) - $(E) "[LD] Linking $@" - $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,--output-def=$(LIBDIR)/$(CONFIG)/address_sorting$(SHARED_VERSION_CORE).def -Wl,--out-implib=$(LIBDIR)/$(CONFIG)/libaddress_sorting$(SHARED_VERSION_CORE)-dll.a -o $(LIBDIR)/$(CONFIG)/address_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBADDRESS_SORTING_OBJS) $(LDLIBS) -else -$(LIBDIR)/$(CONFIG)/libaddress_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBADDRESS_SORTING_OBJS) - $(E) "[LD] Linking $@" - $(Q) mkdir -p `dirname $@` -ifeq ($(SYSTEM),Darwin) - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)address_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libaddress_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBADDRESS_SORTING_OBJS) $(LDLIBS) -else - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libaddress_sorting.so.39 -o $(LIBDIR)/$(CONFIG)/libaddress_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBADDRESS_SORTING_OBJS) $(LDLIBS) - $(Q) ln -sf $(SHARED_PREFIX)address_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libaddress_sorting$(SHARED_VERSION_CORE).so.39 - $(Q) ln -sf $(SHARED_PREFIX)address_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libaddress_sorting$(SHARED_VERSION_CORE).so -endif -endif - -ifneq ($(NO_DEPS),true) --include $(LIBADDRESS_SORTING_OBJS:.o=.dep) -endif -# end of build recipe for library "address_sorting" - - -# start of build recipe for library "gpr" (generated by makelib(lib) template function) -# deps: ['grpc_abseil'] -# transitive_deps: ['grpc_abseil'] -LIBGPR_SRC = \ - src/core/lib/config/config_vars.cc \ - src/core/lib/config/config_vars_non_generated.cc \ - src/core/lib/config/load_config.cc \ - src/core/lib/event_engine/thread_local.cc \ - src/core/lib/gpr/alloc.cc \ - src/core/lib/gpr/android/log.cc \ - src/core/lib/gpr/atm.cc \ - src/core/lib/gpr/iphone/cpu.cc \ - src/core/lib/gpr/linux/cpu.cc \ - src/core/lib/gpr/linux/log.cc \ - src/core/lib/gpr/log.cc \ - src/core/lib/gpr/msys/tmpfile.cc \ - src/core/lib/gpr/posix/cpu.cc \ - src/core/lib/gpr/posix/log.cc \ - src/core/lib/gpr/posix/string.cc \ - src/core/lib/gpr/posix/sync.cc \ - src/core/lib/gpr/posix/time.cc \ - src/core/lib/gpr/posix/tmpfile.cc \ - src/core/lib/gpr/string.cc \ - src/core/lib/gpr/sync.cc \ - src/core/lib/gpr/sync_abseil.cc \ - src/core/lib/gpr/time.cc \ - src/core/lib/gpr/time_precise.cc \ - src/core/lib/gpr/windows/cpu.cc \ - src/core/lib/gpr/windows/log.cc \ - src/core/lib/gpr/windows/string.cc \ - src/core/lib/gpr/windows/string_util.cc \ - src/core/lib/gpr/windows/sync.cc \ - src/core/lib/gpr/windows/time.cc \ - src/core/lib/gpr/windows/tmpfile.cc \ - src/core/lib/gprpp/crash.cc \ - src/core/lib/gprpp/examine_stack.cc \ - src/core/lib/gprpp/fork.cc \ - src/core/lib/gprpp/host_port.cc \ - src/core/lib/gprpp/linux/env.cc \ - src/core/lib/gprpp/mpscq.cc \ - src/core/lib/gprpp/posix/env.cc \ - src/core/lib/gprpp/posix/stat.cc \ - src/core/lib/gprpp/posix/thd.cc \ - src/core/lib/gprpp/strerror.cc \ - src/core/lib/gprpp/tchar.cc \ - src/core/lib/gprpp/time_util.cc \ - src/core/lib/gprpp/windows/env.cc \ - src/core/lib/gprpp/windows/stat.cc \ - src/core/lib/gprpp/windows/thd.cc \ - -PUBLIC_HEADERS_C += \ - include/grpc/impl/call.h \ - include/grpc/impl/codegen/atm.h \ - include/grpc/impl/codegen/atm_gcc_atomic.h \ - include/grpc/impl/codegen/atm_gcc_sync.h \ - include/grpc/impl/codegen/atm_windows.h \ - include/grpc/impl/codegen/fork.h \ - include/grpc/impl/codegen/gpr_types.h \ - include/grpc/impl/codegen/log.h \ - include/grpc/impl/codegen/port_platform.h \ - include/grpc/impl/codegen/sync.h \ - include/grpc/impl/codegen/sync_abseil.h \ - include/grpc/impl/codegen/sync_custom.h \ - include/grpc/impl/codegen/sync_generic.h \ - include/grpc/impl/codegen/sync_posix.h \ - include/grpc/impl/codegen/sync_windows.h \ - include/grpc/support/alloc.h \ - include/grpc/support/atm.h \ - include/grpc/support/atm_gcc_atomic.h \ - include/grpc/support/atm_gcc_sync.h \ - include/grpc/support/atm_windows.h \ - include/grpc/support/cpu.h \ - include/grpc/support/json.h \ - include/grpc/support/log.h \ - include/grpc/support/log_windows.h \ - include/grpc/support/port_platform.h \ - include/grpc/support/string_util.h \ - include/grpc/support/sync.h \ - include/grpc/support/sync_abseil.h \ - include/grpc/support/sync_custom.h \ - include/grpc/support/sync_generic.h \ - include/grpc/support/sync_posix.h \ - include/grpc/support/sync_windows.h \ - include/grpc/support/thd_id.h \ - include/grpc/support/time.h \ - -LIBGPR_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBGPR_SRC)))) - - -# static library for "gpr" -$(LIBDIR)/$(CONFIG)/libgpr.a: $(GRPC_ABSEIL_DEP) $(LIBGPR_OBJS) $(LIBGRPC_ABSEIL_OBJS) - $(E) "[AR] Creating $@" - $(Q) mkdir -p `dirname $@` - $(Q) rm -f $(LIBDIR)/$(CONFIG)/libgpr.a - $(Q) $(AR) $(ARFLAGS) $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBGPR_OBJS) $(LIBGRPC_ABSEIL_OBJS) -ifeq ($(SYSTEM),Darwin) - $(Q) $(RANLIB) $(RANLIBFLAGS) $(LIBDIR)/$(CONFIG)/libgpr.a -endif - -# shared library for "gpr" -ifeq ($(SYSTEM),MINGW32) -$(LIBDIR)/$(CONFIG)/gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBGPR_OBJS) $(GRPC_ABSEIL_DEP) - $(E) "[LD] Linking $@" - $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,--output-def=$(LIBDIR)/$(CONFIG)/gpr$(SHARED_VERSION_CORE).def -Wl,--out-implib=$(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE)-dll.a -o $(LIBDIR)/$(CONFIG)/gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGPR_OBJS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) -else -$(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBGPR_OBJS) $(GRPC_ABSEIL_DEP) - $(E) "[LD] Linking $@" - $(Q) mkdir -p `dirname $@` -ifeq ($(SYSTEM),Darwin) - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGPR_OBJS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) -else - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgpr.so.39 -o $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGPR_OBJS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) - $(Q) ln -sf $(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).so.39 - $(Q) ln -sf $(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).so -endif -endif - -ifneq ($(NO_DEPS),true) --include $(LIBGPR_OBJS:.o=.dep) -endif -# end of build recipe for library "gpr" - - # start of build recipe for library "grpc" (generated by makelib(lib) template function) -# deps: ['upb_json_lib', 'upb_textformat_lib', 're2', 'z', 'grpc_abseil', 'cares', 'gpr', 'libssl', 'address_sorting'] -# transitive_deps: ['address_sorting', 'gpr', 'grpc_abseil', 'cares', 'z', 're2', 'upb_textformat_lib', 'upb_json_lib', 'utf8_range_lib', 'upb_message_lib', 'upb_mem_lib', 'upb_base_lib', 'libssl'] +# deps: ['cares', 'libssl', 'z'] +# transitive_deps: ['cares', 'libssl', 'z'] LIBGRPC_SRC = \ src/core/client_channel/backup_poller.cc \ src/core/client_channel/client_channel_channelz.cc \ @@ -1397,7 +1105,10 @@ LIBGRPC_SRC = \ src/core/lib/compression/compression.cc \ src/core/lib/compression/compression_internal.cc \ src/core/lib/compression/message_compress.cc \ + src/core/lib/config/config_vars.cc \ + src/core/lib/config/config_vars_non_generated.cc \ src/core/lib/config/core_configuration.cc \ + src/core/lib/config/load_config.cc \ src/core/lib/debug/event_log.cc \ src/core/lib/debug/histogram_view.cc \ src/core/lib/debug/stats.cc \ @@ -1435,6 +1146,7 @@ LIBGRPC_SRC = \ src/core/lib/event_engine/slice.cc \ src/core/lib/event_engine/slice_buffer.cc \ src/core/lib/event_engine/tcp_socket_utils.cc \ + src/core/lib/event_engine/thread_local.cc \ src/core/lib/event_engine/thread_pool/thread_count.cc \ src/core/lib/event_engine/thread_pool/thread_pool_factory.cc \ src/core/lib/event_engine/thread_pool/work_stealing_thread_pool.cc \ @@ -1452,16 +1164,57 @@ LIBGRPC_SRC = \ src/core/lib/event_engine/work_queue/basic_work_queue.cc \ src/core/lib/experiments/config.cc \ src/core/lib/experiments/experiments.cc \ + src/core/lib/gpr/alloc.cc \ + src/core/lib/gpr/android/log.cc \ + src/core/lib/gpr/atm.cc \ + src/core/lib/gpr/iphone/cpu.cc \ + src/core/lib/gpr/linux/cpu.cc \ + src/core/lib/gpr/linux/log.cc \ + src/core/lib/gpr/log.cc \ + src/core/lib/gpr/msys/tmpfile.cc \ + src/core/lib/gpr/posix/cpu.cc \ + src/core/lib/gpr/posix/log.cc \ + src/core/lib/gpr/posix/string.cc \ + src/core/lib/gpr/posix/sync.cc \ + src/core/lib/gpr/posix/time.cc \ + src/core/lib/gpr/posix/tmpfile.cc \ + src/core/lib/gpr/string.cc \ + src/core/lib/gpr/sync.cc \ + src/core/lib/gpr/sync_abseil.cc \ + src/core/lib/gpr/time.cc \ + src/core/lib/gpr/time_precise.cc \ + src/core/lib/gpr/windows/cpu.cc \ + src/core/lib/gpr/windows/log.cc \ + src/core/lib/gpr/windows/string.cc \ + src/core/lib/gpr/windows/string_util.cc \ + src/core/lib/gpr/windows/sync.cc \ + src/core/lib/gpr/windows/time.cc \ + src/core/lib/gpr/windows/tmpfile.cc \ + src/core/lib/gprpp/crash.cc \ + src/core/lib/gprpp/examine_stack.cc \ + src/core/lib/gprpp/fork.cc \ + src/core/lib/gprpp/host_port.cc \ + src/core/lib/gprpp/linux/env.cc \ src/core/lib/gprpp/load_file.cc \ + src/core/lib/gprpp/mpscq.cc \ src/core/lib/gprpp/per_cpu.cc \ src/core/lib/gprpp/posix/directory_reader.cc \ + src/core/lib/gprpp/posix/env.cc \ + src/core/lib/gprpp/posix/stat.cc \ + src/core/lib/gprpp/posix/thd.cc \ src/core/lib/gprpp/ref_counted_string.cc \ src/core/lib/gprpp/status_helper.cc \ + src/core/lib/gprpp/strerror.cc \ + src/core/lib/gprpp/tchar.cc \ src/core/lib/gprpp/time.cc \ src/core/lib/gprpp/time_averaged_stats.cc \ + src/core/lib/gprpp/time_util.cc \ src/core/lib/gprpp/uuid_v4.cc \ src/core/lib/gprpp/validation_errors.cc \ src/core/lib/gprpp/windows/directory_reader.cc \ + src/core/lib/gprpp/windows/env.cc \ + src/core/lib/gprpp/windows/stat.cc \ + src/core/lib/gprpp/windows/thd.cc \ src/core/lib/gprpp/work_serializer.cc \ src/core/lib/handshaker/proxy_mapper_registry.cc \ src/core/lib/http/format_request.cc \ @@ -1760,52 +1513,249 @@ LIBGRPC_SRC = \ src/core/tsi/ssl_transport_security_utils.cc \ src/core/tsi/transport_security.cc \ src/core/tsi/transport_security_grpc.cc \ - -PUBLIC_HEADERS_C += \ - include/grpc/byte_buffer.h \ - include/grpc/byte_buffer_reader.h \ - include/grpc/census.h \ - include/grpc/compression.h \ - include/grpc/event_engine/endpoint_config.h \ - include/grpc/event_engine/event_engine.h \ - include/grpc/event_engine/extensible.h \ - include/grpc/event_engine/internal/memory_allocator_impl.h \ - include/grpc/event_engine/internal/slice_cast.h \ - include/grpc/event_engine/memory_allocator.h \ - include/grpc/event_engine/memory_request.h \ - include/grpc/event_engine/port.h \ - include/grpc/event_engine/slice.h \ - include/grpc/event_engine/slice_buffer.h \ - include/grpc/fork.h \ - include/grpc/grpc.h \ - include/grpc/grpc_audit_logging.h \ - include/grpc/grpc_crl_provider.h \ - include/grpc/grpc_posix.h \ - include/grpc/grpc_security.h \ - include/grpc/grpc_security_constants.h \ - include/grpc/impl/call.h \ - include/grpc/impl/channel_arg_names.h \ - include/grpc/impl/codegen/atm.h \ - include/grpc/impl/codegen/atm_gcc_atomic.h \ - include/grpc/impl/codegen/atm_gcc_sync.h \ - include/grpc/impl/codegen/atm_windows.h \ - include/grpc/impl/codegen/byte_buffer.h \ - include/grpc/impl/codegen/byte_buffer_reader.h \ - include/grpc/impl/codegen/compression_types.h \ - include/grpc/impl/codegen/connectivity_state.h \ - include/grpc/impl/codegen/fork.h \ - include/grpc/impl/codegen/gpr_types.h \ - include/grpc/impl/codegen/grpc_types.h \ - include/grpc/impl/codegen/log.h \ - include/grpc/impl/codegen/port_platform.h \ - include/grpc/impl/codegen/propagation_bits.h \ - include/grpc/impl/codegen/slice.h \ - include/grpc/impl/codegen/status.h \ - include/grpc/impl/codegen/sync.h \ - include/grpc/impl/codegen/sync_abseil.h \ - include/grpc/impl/codegen/sync_custom.h \ - include/grpc/impl/codegen/sync_generic.h \ - include/grpc/impl/codegen/sync_posix.h \ + third_party/abseil-cpp/absl/base/internal/cycleclock.cc \ + third_party/abseil-cpp/absl/base/internal/low_level_alloc.cc \ + third_party/abseil-cpp/absl/base/internal/raw_logging.cc \ + third_party/abseil-cpp/absl/base/internal/spinlock.cc \ + third_party/abseil-cpp/absl/base/internal/spinlock_wait.cc \ + third_party/abseil-cpp/absl/base/internal/strerror.cc \ + third_party/abseil-cpp/absl/base/internal/sysinfo.cc \ + third_party/abseil-cpp/absl/base/internal/thread_identity.cc \ + third_party/abseil-cpp/absl/base/internal/throw_delegate.cc \ + third_party/abseil-cpp/absl/base/internal/unscaledcycleclock.cc \ + third_party/abseil-cpp/absl/base/log_severity.cc \ + third_party/abseil-cpp/absl/container/internal/hashtablez_sampler.cc \ + third_party/abseil-cpp/absl/container/internal/hashtablez_sampler_force_weak_definition.cc \ + third_party/abseil-cpp/absl/container/internal/raw_hash_set.cc \ + third_party/abseil-cpp/absl/crc/crc32c.cc \ + third_party/abseil-cpp/absl/crc/internal/cpu_detect.cc \ + third_party/abseil-cpp/absl/crc/internal/crc.cc \ + third_party/abseil-cpp/absl/crc/internal/crc_cord_state.cc \ + third_party/abseil-cpp/absl/crc/internal/crc_memcpy_fallback.cc \ + third_party/abseil-cpp/absl/crc/internal/crc_memcpy_x86_arm_combined.cc \ + third_party/abseil-cpp/absl/crc/internal/crc_non_temporal_memcpy.cc \ + third_party/abseil-cpp/absl/crc/internal/crc_x86_arm_combined.cc \ + third_party/abseil-cpp/absl/debugging/internal/address_is_readable.cc \ + third_party/abseil-cpp/absl/debugging/internal/demangle.cc \ + third_party/abseil-cpp/absl/debugging/internal/elf_mem_image.cc \ + third_party/abseil-cpp/absl/debugging/internal/vdso_support.cc \ + third_party/abseil-cpp/absl/debugging/stacktrace.cc \ + third_party/abseil-cpp/absl/debugging/symbolize.cc \ + third_party/abseil-cpp/absl/flags/commandlineflag.cc \ + third_party/abseil-cpp/absl/flags/internal/commandlineflag.cc \ + third_party/abseil-cpp/absl/flags/internal/flag.cc \ + third_party/abseil-cpp/absl/flags/internal/private_handle_accessor.cc \ + third_party/abseil-cpp/absl/flags/internal/program_name.cc \ + third_party/abseil-cpp/absl/flags/marshalling.cc \ + third_party/abseil-cpp/absl/flags/reflection.cc \ + third_party/abseil-cpp/absl/flags/usage_config.cc \ + third_party/abseil-cpp/absl/hash/internal/city.cc \ + third_party/abseil-cpp/absl/hash/internal/hash.cc \ + third_party/abseil-cpp/absl/hash/internal/low_level_hash.cc \ + third_party/abseil-cpp/absl/numeric/int128.cc \ + third_party/abseil-cpp/absl/profiling/internal/exponential_biased.cc \ + third_party/abseil-cpp/absl/random/discrete_distribution.cc \ + third_party/abseil-cpp/absl/random/gaussian_distribution.cc \ + third_party/abseil-cpp/absl/random/internal/pool_urbg.cc \ + third_party/abseil-cpp/absl/random/internal/randen.cc \ + third_party/abseil-cpp/absl/random/internal/randen_detect.cc \ + third_party/abseil-cpp/absl/random/internal/randen_hwaes.cc \ + third_party/abseil-cpp/absl/random/internal/randen_round_keys.cc \ + third_party/abseil-cpp/absl/random/internal/randen_slow.cc \ + third_party/abseil-cpp/absl/random/internal/seed_material.cc \ + third_party/abseil-cpp/absl/random/seed_gen_exception.cc \ + third_party/abseil-cpp/absl/random/seed_sequences.cc \ + third_party/abseil-cpp/absl/status/internal/status_internal.cc \ + third_party/abseil-cpp/absl/status/status.cc \ + third_party/abseil-cpp/absl/status/status_payload_printer.cc \ + third_party/abseil-cpp/absl/status/statusor.cc \ + third_party/abseil-cpp/absl/strings/ascii.cc \ + third_party/abseil-cpp/absl/strings/charconv.cc \ + third_party/abseil-cpp/absl/strings/cord.cc \ + third_party/abseil-cpp/absl/strings/cord_analysis.cc \ + third_party/abseil-cpp/absl/strings/cord_buffer.cc \ + third_party/abseil-cpp/absl/strings/escaping.cc \ + third_party/abseil-cpp/absl/strings/internal/charconv_bigint.cc \ + third_party/abseil-cpp/absl/strings/internal/charconv_parse.cc \ + third_party/abseil-cpp/absl/strings/internal/cord_internal.cc \ + third_party/abseil-cpp/absl/strings/internal/cord_rep_btree.cc \ + third_party/abseil-cpp/absl/strings/internal/cord_rep_btree_navigator.cc \ + third_party/abseil-cpp/absl/strings/internal/cord_rep_btree_reader.cc \ + third_party/abseil-cpp/absl/strings/internal/cord_rep_consume.cc \ + third_party/abseil-cpp/absl/strings/internal/cord_rep_crc.cc \ + third_party/abseil-cpp/absl/strings/internal/cordz_functions.cc \ + third_party/abseil-cpp/absl/strings/internal/cordz_handle.cc \ + third_party/abseil-cpp/absl/strings/internal/cordz_info.cc \ + third_party/abseil-cpp/absl/strings/internal/damerau_levenshtein_distance.cc \ + third_party/abseil-cpp/absl/strings/internal/escaping.cc \ + third_party/abseil-cpp/absl/strings/internal/memutil.cc \ + third_party/abseil-cpp/absl/strings/internal/ostringstream.cc \ + third_party/abseil-cpp/absl/strings/internal/str_format/arg.cc \ + third_party/abseil-cpp/absl/strings/internal/str_format/bind.cc \ + third_party/abseil-cpp/absl/strings/internal/str_format/extension.cc \ + third_party/abseil-cpp/absl/strings/internal/str_format/float_conversion.cc \ + third_party/abseil-cpp/absl/strings/internal/str_format/output.cc \ + third_party/abseil-cpp/absl/strings/internal/str_format/parser.cc \ + third_party/abseil-cpp/absl/strings/internal/stringify_sink.cc \ + third_party/abseil-cpp/absl/strings/internal/utf8.cc \ + third_party/abseil-cpp/absl/strings/match.cc \ + third_party/abseil-cpp/absl/strings/numbers.cc \ + third_party/abseil-cpp/absl/strings/str_cat.cc \ + third_party/abseil-cpp/absl/strings/str_replace.cc \ + third_party/abseil-cpp/absl/strings/str_split.cc \ + third_party/abseil-cpp/absl/strings/string_view.cc \ + third_party/abseil-cpp/absl/strings/substitute.cc \ + third_party/abseil-cpp/absl/synchronization/barrier.cc \ + third_party/abseil-cpp/absl/synchronization/blocking_counter.cc \ + third_party/abseil-cpp/absl/synchronization/internal/create_thread_identity.cc \ + third_party/abseil-cpp/absl/synchronization/internal/futex_waiter.cc \ + third_party/abseil-cpp/absl/synchronization/internal/graphcycles.cc \ + third_party/abseil-cpp/absl/synchronization/internal/kernel_timeout.cc \ + third_party/abseil-cpp/absl/synchronization/internal/per_thread_sem.cc \ + third_party/abseil-cpp/absl/synchronization/internal/pthread_waiter.cc \ + third_party/abseil-cpp/absl/synchronization/internal/sem_waiter.cc \ + third_party/abseil-cpp/absl/synchronization/internal/stdcpp_waiter.cc \ + third_party/abseil-cpp/absl/synchronization/internal/waiter_base.cc \ + third_party/abseil-cpp/absl/synchronization/internal/win32_waiter.cc \ + third_party/abseil-cpp/absl/synchronization/mutex.cc \ + third_party/abseil-cpp/absl/synchronization/notification.cc \ + third_party/abseil-cpp/absl/time/civil_time.cc \ + third_party/abseil-cpp/absl/time/clock.cc \ + third_party/abseil-cpp/absl/time/duration.cc \ + third_party/abseil-cpp/absl/time/format.cc \ + third_party/abseil-cpp/absl/time/internal/cctz/src/civil_time_detail.cc \ + third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_fixed.cc \ + third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_format.cc \ + third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_if.cc \ + third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_impl.cc \ + third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_info.cc \ + third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_libc.cc \ + third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_lookup.cc \ + third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_posix.cc \ + third_party/abseil-cpp/absl/time/internal/cctz/src/zone_info_source.cc \ + third_party/abseil-cpp/absl/time/time.cc \ + third_party/abseil-cpp/absl/types/bad_optional_access.cc \ + third_party/abseil-cpp/absl/types/bad_variant_access.cc \ + third_party/address_sorting/address_sorting.c \ + third_party/address_sorting/address_sorting_posix.c \ + third_party/address_sorting/address_sorting_windows.c \ + third_party/re2/re2/bitstate.cc \ + third_party/re2/re2/compile.cc \ + third_party/re2/re2/dfa.cc \ + third_party/re2/re2/filtered_re2.cc \ + third_party/re2/re2/mimics_pcre.cc \ + third_party/re2/re2/nfa.cc \ + third_party/re2/re2/onepass.cc \ + third_party/re2/re2/parse.cc \ + third_party/re2/re2/perl_groups.cc \ + third_party/re2/re2/prefilter.cc \ + third_party/re2/re2/prefilter_tree.cc \ + third_party/re2/re2/prog.cc \ + third_party/re2/re2/re2.cc \ + third_party/re2/re2/regexp.cc \ + third_party/re2/re2/set.cc \ + third_party/re2/re2/simplify.cc \ + third_party/re2/re2/stringpiece.cc \ + third_party/re2/re2/tostring.cc \ + third_party/re2/re2/unicode_casefold.cc \ + third_party/re2/re2/unicode_groups.cc \ + third_party/re2/util/rune.cc \ + third_party/re2/util/strutil.cc \ + third_party/upb/upb/base/status.c \ + third_party/upb/upb/hash/common.c \ + third_party/upb/upb/json/decode.c \ + third_party/upb/upb/json/encode.c \ + third_party/upb/upb/lex/atoi.c \ + third_party/upb/upb/lex/round_trip.c \ + third_party/upb/upb/lex/strtod.c \ + third_party/upb/upb/lex/unicode.c \ + third_party/upb/upb/mem/alloc.c \ + third_party/upb/upb/mem/arena.c \ + third_party/upb/upb/message/accessors.c \ + third_party/upb/upb/message/array.c \ + third_party/upb/upb/message/map.c \ + third_party/upb/upb/message/map_sorter.c \ + third_party/upb/upb/message/message.c \ + third_party/upb/upb/mini_descriptor/build_enum.c \ + third_party/upb/upb/mini_descriptor/decode.c \ + third_party/upb/upb/mini_descriptor/internal/base92.c \ + third_party/upb/upb/mini_descriptor/internal/encode.c \ + third_party/upb/upb/mini_descriptor/link.c \ + third_party/upb/upb/mini_table/extension_registry.c \ + third_party/upb/upb/mini_table/internal/message.c \ + third_party/upb/upb/mini_table/message.c \ + third_party/upb/upb/reflection/def_pool.c \ + third_party/upb/upb/reflection/def_type.c \ + third_party/upb/upb/reflection/desc_state.c \ + third_party/upb/upb/reflection/enum_def.c \ + third_party/upb/upb/reflection/enum_reserved_range.c \ + third_party/upb/upb/reflection/enum_value_def.c \ + third_party/upb/upb/reflection/extension_range.c \ + third_party/upb/upb/reflection/field_def.c \ + third_party/upb/upb/reflection/file_def.c \ + third_party/upb/upb/reflection/internal/def_builder.c \ + third_party/upb/upb/reflection/internal/strdup2.c \ + third_party/upb/upb/reflection/message.c \ + third_party/upb/upb/reflection/message_def.c \ + third_party/upb/upb/reflection/message_reserved_range.c \ + third_party/upb/upb/reflection/method_def.c \ + third_party/upb/upb/reflection/oneof_def.c \ + third_party/upb/upb/reflection/service_def.c \ + third_party/upb/upb/text/encode.c \ + third_party/upb/upb/wire/decode.c \ + third_party/upb/upb/wire/decode_fast.c \ + third_party/upb/upb/wire/encode.c \ + third_party/upb/upb/wire/eps_copy_input_stream.c \ + third_party/upb/upb/wire/reader.c \ + third_party/utf8_range/naive.c \ + third_party/utf8_range/range2-neon.c \ + third_party/utf8_range/range2-sse.c \ + +PUBLIC_HEADERS_C += \ + include/grpc/byte_buffer.h \ + include/grpc/byte_buffer_reader.h \ + include/grpc/census.h \ + include/grpc/compression.h \ + include/grpc/event_engine/endpoint_config.h \ + include/grpc/event_engine/event_engine.h \ + include/grpc/event_engine/extensible.h \ + include/grpc/event_engine/internal/memory_allocator_impl.h \ + include/grpc/event_engine/internal/slice_cast.h \ + include/grpc/event_engine/memory_allocator.h \ + include/grpc/event_engine/memory_request.h \ + include/grpc/event_engine/port.h \ + include/grpc/event_engine/slice.h \ + include/grpc/event_engine/slice_buffer.h \ + include/grpc/fork.h \ + include/grpc/grpc.h \ + include/grpc/grpc_audit_logging.h \ + include/grpc/grpc_crl_provider.h \ + include/grpc/grpc_posix.h \ + include/grpc/grpc_security.h \ + include/grpc/grpc_security_constants.h \ + include/grpc/impl/call.h \ + include/grpc/impl/channel_arg_names.h \ + include/grpc/impl/codegen/atm.h \ + include/grpc/impl/codegen/atm_gcc_atomic.h \ + include/grpc/impl/codegen/atm_gcc_sync.h \ + include/grpc/impl/codegen/atm_windows.h \ + include/grpc/impl/codegen/byte_buffer.h \ + include/grpc/impl/codegen/byte_buffer_reader.h \ + include/grpc/impl/codegen/compression_types.h \ + include/grpc/impl/codegen/connectivity_state.h \ + include/grpc/impl/codegen/fork.h \ + include/grpc/impl/codegen/gpr_types.h \ + include/grpc/impl/codegen/grpc_types.h \ + include/grpc/impl/codegen/log.h \ + include/grpc/impl/codegen/port_platform.h \ + include/grpc/impl/codegen/propagation_bits.h \ + include/grpc/impl/codegen/slice.h \ + include/grpc/impl/codegen/status.h \ + include/grpc/impl/codegen/sync.h \ + include/grpc/impl/codegen/sync_abseil.h \ + include/grpc/impl/codegen/sync_custom.h \ + include/grpc/impl/codegen/sync_generic.h \ + include/grpc/impl/codegen/sync_posix.h \ include/grpc/impl/codegen/sync_windows.h \ include/grpc/impl/compression_types.h \ include/grpc/impl/connectivity_state.h \ @@ -1850,29 +1800,29 @@ $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE else # static library for "grpc" -$(LIBDIR)/$(CONFIG)/libgrpc.a: $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libgpr.a $(GRPC_ABSEIL_DEP) $(LIBDIR)/$(CONFIG)/libcares.a $(ZLIB_DEP) $(LIBDIR)/$(CONFIG)/libre2.a $(LIBDIR)/$(CONFIG)/libupb_textformat_lib.a $(LIBDIR)/$(CONFIG)/libupb_json_lib.a $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(OPENSSL_DEP) $(LIBGRPC_OBJS) $(LIBADDRESS_SORTING_OBJS) $(LIBGPR_OBJS) $(LIBGRPC_ABSEIL_OBJS) $(LIBCARES_OBJS) $(ZLIB_MERGE_OBJS) $(LIBRE2_OBJS) $(LIBUPB_TEXTFORMAT_LIB_OBJS) $(LIBUPB_JSON_LIB_OBJS) $(LIBUTF8_RANGE_LIB_OBJS) $(LIBUPB_MESSAGE_LIB_OBJS) $(LIBUPB_MEM_LIB_OBJS) $(LIBUPB_BASE_LIB_OBJS) $(OPENSSL_MERGE_OBJS) +$(LIBDIR)/$(CONFIG)/libgrpc.a: $(LIBDIR)/$(CONFIG)/libcares.a $(OPENSSL_DEP) $(ZLIB_DEP) $(LIBGRPC_OBJS) $(LIBCARES_OBJS) $(OPENSSL_MERGE_OBJS) $(ZLIB_MERGE_OBJS) $(E) "[AR] Creating $@" $(Q) mkdir -p `dirname $@` $(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc.a - $(Q) $(AR) $(ARFLAGS) $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBGRPC_OBJS) $(LIBADDRESS_SORTING_OBJS) $(LIBGPR_OBJS) $(LIBGRPC_ABSEIL_OBJS) $(LIBCARES_OBJS) $(ZLIB_MERGE_OBJS) $(LIBRE2_OBJS) $(LIBUPB_TEXTFORMAT_LIB_OBJS) $(LIBUPB_JSON_LIB_OBJS) $(LIBUTF8_RANGE_LIB_OBJS) $(LIBUPB_MESSAGE_LIB_OBJS) $(LIBUPB_MEM_LIB_OBJS) $(LIBUPB_BASE_LIB_OBJS) $(OPENSSL_MERGE_OBJS) + $(Q) $(AR) $(ARFLAGS) $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBGRPC_OBJS) $(LIBCARES_OBJS) $(OPENSSL_MERGE_OBJS) $(ZLIB_MERGE_OBJS) ifeq ($(SYSTEM),Darwin) $(Q) $(RANLIB) $(RANLIBFLAGS) $(LIBDIR)/$(CONFIG)/libgrpc.a endif # shared library for "grpc" ifeq ($(SYSTEM),MINGW32) -$(LIBDIR)/$(CONFIG)/grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBGRPC_OBJS) $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libgpr.a $(GRPC_ABSEIL_DEP) $(LIBDIR)/$(CONFIG)/libcares.a $(ZLIB_DEP) $(LIBDIR)/$(CONFIG)/libre2.a $(LIBDIR)/$(CONFIG)/libupb_textformat_lib.a $(LIBDIR)/$(CONFIG)/libupb_json_lib.a $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(OPENSSL_DEP) +$(LIBDIR)/$(CONFIG)/grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBGRPC_OBJS) $(LIBDIR)/$(CONFIG)/libcares.a $(OPENSSL_DEP) $(ZLIB_DEP) $(E) "[LD] Linking $@" $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,--output-def=$(LIBDIR)/$(CONFIG)/grpc$(SHARED_VERSION_CORE).def -Wl,--out-implib=$(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE)-dll.a -o $(LIBDIR)/$(CONFIG)/grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_OBJS) $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libgpr.a $(GRPC_ABSEIL_MERGE_LIBS) $(LIBDIR)/$(CONFIG)/libcares.a $(ZLIB_MERGE_LIBS) $(LIBDIR)/$(CONFIG)/libre2.a $(LIBDIR)/$(CONFIG)/libupb_textformat_lib.a $(LIBDIR)/$(CONFIG)/libupb_json_lib.a $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(OPENSSL_MERGE_LIBS) $(LDLIBS_SECURE) $(LDLIBS) + $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,--output-def=$(LIBDIR)/$(CONFIG)/grpc$(SHARED_VERSION_CORE).def -Wl,--out-implib=$(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE)-dll.a -o $(LIBDIR)/$(CONFIG)/grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_OBJS) $(LIBDIR)/$(CONFIG)/libcares.a $(OPENSSL_MERGE_LIBS) $(ZLIB_MERGE_LIBS) $(LDLIBS_SECURE) $(LDLIBS) else -$(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBGRPC_OBJS) $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libgpr.a $(GRPC_ABSEIL_DEP) $(LIBDIR)/$(CONFIG)/libcares.a $(ZLIB_DEP) $(LIBDIR)/$(CONFIG)/libre2.a $(LIBDIR)/$(CONFIG)/libupb_textformat_lib.a $(LIBDIR)/$(CONFIG)/libupb_json_lib.a $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(OPENSSL_DEP) +$(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBGRPC_OBJS) $(LIBDIR)/$(CONFIG)/libcares.a $(OPENSSL_DEP) $(ZLIB_DEP) $(E) "[LD] Linking $@" $(Q) mkdir -p `dirname $@` ifeq ($(SYSTEM),Darwin) - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_OBJS) $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libgpr.a $(GRPC_ABSEIL_MERGE_LIBS) $(LIBDIR)/$(CONFIG)/libcares.a $(ZLIB_MERGE_LIBS) $(LIBDIR)/$(CONFIG)/libre2.a $(LIBDIR)/$(CONFIG)/libupb_textformat_lib.a $(LIBDIR)/$(CONFIG)/libupb_json_lib.a $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(OPENSSL_MERGE_LIBS) $(LDLIBS_SECURE) $(LDLIBS) + $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_OBJS) $(LIBDIR)/$(CONFIG)/libcares.a $(OPENSSL_MERGE_LIBS) $(ZLIB_MERGE_LIBS) $(LDLIBS_SECURE) $(LDLIBS) else - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgrpc.so.39 -o $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_OBJS) $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libgpr.a $(GRPC_ABSEIL_MERGE_LIBS) $(LIBDIR)/$(CONFIG)/libcares.a $(ZLIB_MERGE_LIBS) $(LIBDIR)/$(CONFIG)/libre2.a $(LIBDIR)/$(CONFIG)/libupb_textformat_lib.a $(LIBDIR)/$(CONFIG)/libupb_json_lib.a $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(OPENSSL_MERGE_LIBS) $(LDLIBS_SECURE) $(LDLIBS) + $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgrpc.so.39 -o $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_OBJS) $(LIBDIR)/$(CONFIG)/libcares.a $(OPENSSL_MERGE_LIBS) $(ZLIB_MERGE_LIBS) $(LDLIBS_SECURE) $(LDLIBS) $(Q) ln -sf $(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).so.39 $(Q) ln -sf $(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).so endif @@ -1888,994 +1838,6 @@ endif # end of build recipe for library "grpc" -# start of build recipe for library "grpc_unsecure" (generated by makelib(lib) template function) -# deps: ['upb_message_lib', 'utf8_range_lib', 'z', 'grpc_abseil', 'cares', 'gpr', 'address_sorting'] -# transitive_deps: ['address_sorting', 'gpr', 'grpc_abseil', 'cares', 'z', 'utf8_range_lib', 'upb_message_lib', 'upb_mem_lib', 'upb_base_lib'] -LIBGRPC_UNSECURE_SRC = \ - src/core/client_channel/backup_poller.cc \ - src/core/client_channel/client_channel_channelz.cc \ - src/core/client_channel/client_channel_factory.cc \ - src/core/client_channel/client_channel_filter.cc \ - src/core/client_channel/client_channel_plugin.cc \ - src/core/client_channel/client_channel_service_config.cc \ - src/core/client_channel/config_selector.cc \ - src/core/client_channel/dynamic_filters.cc \ - src/core/client_channel/global_subchannel_pool.cc \ - src/core/client_channel/http_proxy_mapper.cc \ - src/core/client_channel/local_subchannel_pool.cc \ - src/core/client_channel/retry_filter.cc \ - src/core/client_channel/retry_filter_legacy_call_data.cc \ - src/core/client_channel/retry_service_config.cc \ - src/core/client_channel/retry_throttle.cc \ - src/core/client_channel/subchannel.cc \ - src/core/client_channel/subchannel_pool_interface.cc \ - src/core/client_channel/subchannel_stream_client.cc \ - src/core/ext/filters/backend_metrics/backend_metric_filter.cc \ - src/core/ext/filters/census/grpc_context.cc \ - src/core/ext/filters/channel_idle/channel_idle_filter.cc \ - src/core/ext/filters/channel_idle/idle_filter_state.cc \ - src/core/ext/filters/channel_idle/legacy_channel_idle_filter.cc \ - src/core/ext/filters/deadline/deadline_filter.cc \ - src/core/ext/filters/fault_injection/fault_injection_filter.cc \ - src/core/ext/filters/fault_injection/fault_injection_service_config_parser.cc \ - src/core/ext/filters/http/client/http_client_filter.cc \ - src/core/ext/filters/http/client_authority_filter.cc \ - src/core/ext/filters/http/http_filters_plugin.cc \ - src/core/ext/filters/http/message_compress/compression_filter.cc \ - src/core/ext/filters/http/message_compress/legacy_compression_filter.cc \ - src/core/ext/filters/http/server/http_server_filter.cc \ - src/core/ext/filters/message_size/message_size_filter.cc \ - src/core/ext/transport/chttp2/client/chttp2_connector.cc \ - src/core/ext/transport/chttp2/server/chttp2_server.cc \ - src/core/ext/transport/chttp2/transport/bin_decoder.cc \ - src/core/ext/transport/chttp2/transport/bin_encoder.cc \ - src/core/ext/transport/chttp2/transport/chttp2_transport.cc \ - src/core/ext/transport/chttp2/transport/decode_huff.cc \ - src/core/ext/transport/chttp2/transport/flow_control.cc \ - src/core/ext/transport/chttp2/transport/frame.cc \ - src/core/ext/transport/chttp2/transport/frame_data.cc \ - src/core/ext/transport/chttp2/transport/frame_goaway.cc \ - src/core/ext/transport/chttp2/transport/frame_ping.cc \ - src/core/ext/transport/chttp2/transport/frame_rst_stream.cc \ - src/core/ext/transport/chttp2/transport/frame_settings.cc \ - src/core/ext/transport/chttp2/transport/frame_window_update.cc \ - src/core/ext/transport/chttp2/transport/hpack_encoder.cc \ - src/core/ext/transport/chttp2/transport/hpack_encoder_table.cc \ - src/core/ext/transport/chttp2/transport/hpack_parse_result.cc \ - src/core/ext/transport/chttp2/transport/hpack_parser.cc \ - src/core/ext/transport/chttp2/transport/hpack_parser_table.cc \ - src/core/ext/transport/chttp2/transport/http2_settings.cc \ - src/core/ext/transport/chttp2/transport/http_trace.cc \ - src/core/ext/transport/chttp2/transport/huffsyms.cc \ - src/core/ext/transport/chttp2/transport/max_concurrent_streams_policy.cc \ - src/core/ext/transport/chttp2/transport/parsing.cc \ - src/core/ext/transport/chttp2/transport/ping_abuse_policy.cc \ - src/core/ext/transport/chttp2/transport/ping_callbacks.cc \ - src/core/ext/transport/chttp2/transport/ping_rate_policy.cc \ - src/core/ext/transport/chttp2/transport/stream_lists.cc \ - src/core/ext/transport/chttp2/transport/varint.cc \ - src/core/ext/transport/chttp2/transport/write_size_policy.cc \ - src/core/ext/transport/chttp2/transport/writing.cc \ - src/core/ext/transport/inproc/inproc_plugin.cc \ - src/core/ext/transport/inproc/inproc_transport.cc \ - src/core/ext/transport/inproc/legacy_inproc_transport.cc \ - src/core/ext/upb-gen/google/api/annotations.upb_minitable.c \ - src/core/ext/upb-gen/google/api/http.upb_minitable.c \ - src/core/ext/upb-gen/google/protobuf/any.upb_minitable.c \ - src/core/ext/upb-gen/google/protobuf/descriptor.upb_minitable.c \ - src/core/ext/upb-gen/google/protobuf/duration.upb_minitable.c \ - src/core/ext/upb-gen/google/protobuf/empty.upb_minitable.c \ - src/core/ext/upb-gen/google/protobuf/struct.upb_minitable.c \ - src/core/ext/upb-gen/google/protobuf/timestamp.upb_minitable.c \ - src/core/ext/upb-gen/google/protobuf/wrappers.upb_minitable.c \ - src/core/ext/upb-gen/google/rpc/status.upb_minitable.c \ - src/core/ext/upb-gen/src/proto/grpc/gcp/altscontext.upb_minitable.c \ - src/core/ext/upb-gen/src/proto/grpc/gcp/handshaker.upb_minitable.c \ - src/core/ext/upb-gen/src/proto/grpc/gcp/transport_security_common.upb_minitable.c \ - src/core/ext/upb-gen/src/proto/grpc/health/v1/health.upb_minitable.c \ - src/core/ext/upb-gen/src/proto/grpc/lb/v1/load_balancer.upb_minitable.c \ - src/core/ext/upb-gen/src/proto/grpc/lookup/v1/rls.upb_minitable.c \ - src/core/ext/upb-gen/validate/validate.upb_minitable.c \ - src/core/ext/upb-gen/xds/data/orca/v3/orca_load_report.upb_minitable.c \ - src/core/ext/upb-gen/xds/service/orca/v3/orca.upb_minitable.c \ - src/core/lib/address_utils/parse_address.cc \ - src/core/lib/address_utils/sockaddr_utils.cc \ - src/core/lib/backoff/backoff.cc \ - src/core/lib/backoff/random_early_detection.cc \ - src/core/lib/channel/call_tracer.cc \ - src/core/lib/channel/channel_args.cc \ - src/core/lib/channel/channel_args_preconditioning.cc \ - src/core/lib/channel/channel_stack.cc \ - src/core/lib/channel/channel_stack_builder.cc \ - src/core/lib/channel/channel_stack_builder_impl.cc \ - src/core/lib/channel/channel_stack_trace.cc \ - src/core/lib/channel/channel_trace.cc \ - src/core/lib/channel/channelz.cc \ - src/core/lib/channel/channelz_registry.cc \ - src/core/lib/channel/connected_channel.cc \ - src/core/lib/channel/metrics.cc \ - src/core/lib/channel/promise_based_filter.cc \ - src/core/lib/channel/server_call_tracer_filter.cc \ - src/core/lib/channel/status_util.cc \ - src/core/lib/compression/compression.cc \ - src/core/lib/compression/compression_internal.cc \ - src/core/lib/compression/message_compress.cc \ - src/core/lib/config/core_configuration.cc \ - src/core/lib/debug/event_log.cc \ - src/core/lib/debug/histogram_view.cc \ - src/core/lib/debug/stats.cc \ - src/core/lib/debug/stats_data.cc \ - src/core/lib/debug/trace.cc \ - src/core/lib/event_engine/ares_resolver.cc \ - src/core/lib/event_engine/cf_engine/cf_engine.cc \ - src/core/lib/event_engine/cf_engine/cfstream_endpoint.cc \ - src/core/lib/event_engine/cf_engine/dns_service_resolver.cc \ - src/core/lib/event_engine/channel_args_endpoint_config.cc \ - src/core/lib/event_engine/default_event_engine.cc \ - src/core/lib/event_engine/default_event_engine_factory.cc \ - src/core/lib/event_engine/event_engine.cc \ - src/core/lib/event_engine/forkable.cc \ - src/core/lib/event_engine/posix_engine/ev_epoll1_linux.cc \ - src/core/lib/event_engine/posix_engine/ev_poll_posix.cc \ - src/core/lib/event_engine/posix_engine/event_poller_posix_default.cc \ - src/core/lib/event_engine/posix_engine/internal_errqueue.cc \ - src/core/lib/event_engine/posix_engine/lockfree_event.cc \ - src/core/lib/event_engine/posix_engine/native_posix_dns_resolver.cc \ - src/core/lib/event_engine/posix_engine/posix_endpoint.cc \ - src/core/lib/event_engine/posix_engine/posix_engine.cc \ - src/core/lib/event_engine/posix_engine/posix_engine_listener.cc \ - src/core/lib/event_engine/posix_engine/posix_engine_listener_utils.cc \ - src/core/lib/event_engine/posix_engine/tcp_socket_utils.cc \ - src/core/lib/event_engine/posix_engine/timer.cc \ - src/core/lib/event_engine/posix_engine/timer_heap.cc \ - src/core/lib/event_engine/posix_engine/timer_manager.cc \ - src/core/lib/event_engine/posix_engine/traced_buffer_list.cc \ - src/core/lib/event_engine/posix_engine/wakeup_fd_eventfd.cc \ - src/core/lib/event_engine/posix_engine/wakeup_fd_pipe.cc \ - src/core/lib/event_engine/posix_engine/wakeup_fd_posix_default.cc \ - src/core/lib/event_engine/resolved_address.cc \ - src/core/lib/event_engine/shim.cc \ - src/core/lib/event_engine/slice.cc \ - src/core/lib/event_engine/slice_buffer.cc \ - src/core/lib/event_engine/tcp_socket_utils.cc \ - src/core/lib/event_engine/thread_pool/thread_count.cc \ - src/core/lib/event_engine/thread_pool/thread_pool_factory.cc \ - src/core/lib/event_engine/thread_pool/work_stealing_thread_pool.cc \ - src/core/lib/event_engine/thready_event_engine/thready_event_engine.cc \ - src/core/lib/event_engine/time_util.cc \ - src/core/lib/event_engine/trace.cc \ - src/core/lib/event_engine/utils.cc \ - src/core/lib/event_engine/windows/grpc_polled_fd_windows.cc \ - src/core/lib/event_engine/windows/iocp.cc \ - src/core/lib/event_engine/windows/native_windows_dns_resolver.cc \ - src/core/lib/event_engine/windows/win_socket.cc \ - src/core/lib/event_engine/windows/windows_endpoint.cc \ - src/core/lib/event_engine/windows/windows_engine.cc \ - src/core/lib/event_engine/windows/windows_listener.cc \ - src/core/lib/event_engine/work_queue/basic_work_queue.cc \ - src/core/lib/experiments/config.cc \ - src/core/lib/experiments/experiments.cc \ - src/core/lib/gprpp/load_file.cc \ - src/core/lib/gprpp/per_cpu.cc \ - src/core/lib/gprpp/ref_counted_string.cc \ - src/core/lib/gprpp/status_helper.cc \ - src/core/lib/gprpp/time.cc \ - src/core/lib/gprpp/time_averaged_stats.cc \ - src/core/lib/gprpp/uuid_v4.cc \ - src/core/lib/gprpp/validation_errors.cc \ - src/core/lib/gprpp/work_serializer.cc \ - src/core/lib/handshaker/proxy_mapper_registry.cc \ - src/core/lib/http/format_request.cc \ - src/core/lib/http/httpcli.cc \ - src/core/lib/http/parser.cc \ - src/core/lib/iomgr/buffer_list.cc \ - src/core/lib/iomgr/call_combiner.cc \ - src/core/lib/iomgr/cfstream_handle.cc \ - src/core/lib/iomgr/closure.cc \ - src/core/lib/iomgr/combiner.cc \ - src/core/lib/iomgr/dualstack_socket_posix.cc \ - src/core/lib/iomgr/endpoint.cc \ - src/core/lib/iomgr/endpoint_cfstream.cc \ - src/core/lib/iomgr/endpoint_pair_posix.cc \ - src/core/lib/iomgr/endpoint_pair_windows.cc \ - src/core/lib/iomgr/error.cc \ - src/core/lib/iomgr/error_cfstream.cc \ - src/core/lib/iomgr/ev_apple.cc \ - src/core/lib/iomgr/ev_epoll1_linux.cc \ - src/core/lib/iomgr/ev_poll_posix.cc \ - src/core/lib/iomgr/ev_posix.cc \ - src/core/lib/iomgr/ev_windows.cc \ - src/core/lib/iomgr/event_engine_shims/closure.cc \ - src/core/lib/iomgr/event_engine_shims/endpoint.cc \ - src/core/lib/iomgr/event_engine_shims/tcp_client.cc \ - src/core/lib/iomgr/exec_ctx.cc \ - src/core/lib/iomgr/executor.cc \ - src/core/lib/iomgr/fork_posix.cc \ - src/core/lib/iomgr/fork_windows.cc \ - src/core/lib/iomgr/gethostname_fallback.cc \ - src/core/lib/iomgr/gethostname_host_name_max.cc \ - src/core/lib/iomgr/gethostname_sysconf.cc \ - src/core/lib/iomgr/grpc_if_nametoindex_posix.cc \ - src/core/lib/iomgr/grpc_if_nametoindex_unsupported.cc \ - src/core/lib/iomgr/internal_errqueue.cc \ - src/core/lib/iomgr/iocp_windows.cc \ - src/core/lib/iomgr/iomgr.cc \ - src/core/lib/iomgr/iomgr_internal.cc \ - src/core/lib/iomgr/iomgr_posix.cc \ - src/core/lib/iomgr/iomgr_posix_cfstream.cc \ - src/core/lib/iomgr/iomgr_windows.cc \ - src/core/lib/iomgr/lockfree_event.cc \ - src/core/lib/iomgr/polling_entity.cc \ - src/core/lib/iomgr/pollset.cc \ - src/core/lib/iomgr/pollset_set.cc \ - src/core/lib/iomgr/pollset_set_windows.cc \ - src/core/lib/iomgr/pollset_windows.cc \ - src/core/lib/iomgr/resolve_address.cc \ - src/core/lib/iomgr/resolve_address_posix.cc \ - src/core/lib/iomgr/resolve_address_windows.cc \ - src/core/lib/iomgr/sockaddr_utils_posix.cc \ - src/core/lib/iomgr/socket_factory_posix.cc \ - src/core/lib/iomgr/socket_mutator.cc \ - src/core/lib/iomgr/socket_utils_common_posix.cc \ - src/core/lib/iomgr/socket_utils_linux.cc \ - src/core/lib/iomgr/socket_utils_posix.cc \ - src/core/lib/iomgr/socket_utils_windows.cc \ - src/core/lib/iomgr/socket_windows.cc \ - src/core/lib/iomgr/systemd_utils.cc \ - src/core/lib/iomgr/tcp_client.cc \ - src/core/lib/iomgr/tcp_client_cfstream.cc \ - src/core/lib/iomgr/tcp_client_posix.cc \ - src/core/lib/iomgr/tcp_client_windows.cc \ - src/core/lib/iomgr/tcp_posix.cc \ - src/core/lib/iomgr/tcp_server.cc \ - src/core/lib/iomgr/tcp_server_posix.cc \ - src/core/lib/iomgr/tcp_server_utils_posix_common.cc \ - src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.cc \ - src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.cc \ - src/core/lib/iomgr/tcp_server_windows.cc \ - src/core/lib/iomgr/tcp_windows.cc \ - src/core/lib/iomgr/timer.cc \ - src/core/lib/iomgr/timer_generic.cc \ - src/core/lib/iomgr/timer_heap.cc \ - src/core/lib/iomgr/timer_manager.cc \ - src/core/lib/iomgr/unix_sockets_posix.cc \ - src/core/lib/iomgr/unix_sockets_posix_noop.cc \ - src/core/lib/iomgr/vsock.cc \ - src/core/lib/iomgr/wakeup_fd_eventfd.cc \ - src/core/lib/iomgr/wakeup_fd_nospecial.cc \ - src/core/lib/iomgr/wakeup_fd_pipe.cc \ - src/core/lib/iomgr/wakeup_fd_posix.cc \ - src/core/lib/json/json_object_loader.cc \ - src/core/lib/json/json_reader.cc \ - src/core/lib/json/json_writer.cc \ - src/core/lib/promise/activity.cc \ - src/core/lib/promise/party.cc \ - src/core/lib/promise/sleep.cc \ - src/core/lib/promise/trace.cc \ - src/core/lib/resource_quota/api.cc \ - src/core/lib/resource_quota/arena.cc \ - src/core/lib/resource_quota/memory_quota.cc \ - src/core/lib/resource_quota/periodic_update.cc \ - src/core/lib/resource_quota/resource_quota.cc \ - src/core/lib/resource_quota/thread_quota.cc \ - src/core/lib/resource_quota/trace.cc \ - src/core/lib/security/authorization/authorization_policy_provider_vtable.cc \ - src/core/lib/security/authorization/evaluate_args.cc \ - src/core/lib/security/authorization/grpc_server_authz_filter.cc \ - src/core/lib/security/certificate_provider/certificate_provider_registry.cc \ - src/core/lib/security/context/security_context.cc \ - src/core/lib/security/credentials/alts/check_gcp_environment.cc \ - src/core/lib/security/credentials/alts/check_gcp_environment_linux.cc \ - src/core/lib/security/credentials/alts/check_gcp_environment_no_op.cc \ - src/core/lib/security/credentials/alts/check_gcp_environment_windows.cc \ - src/core/lib/security/credentials/alts/grpc_alts_credentials_client_options.cc \ - src/core/lib/security/credentials/alts/grpc_alts_credentials_options.cc \ - src/core/lib/security/credentials/alts/grpc_alts_credentials_server_options.cc \ - src/core/lib/security/credentials/call_creds_util.cc \ - src/core/lib/security/credentials/composite/composite_credentials.cc \ - src/core/lib/security/credentials/credentials.cc \ - src/core/lib/security/credentials/fake/fake_credentials.cc \ - src/core/lib/security/credentials/insecure/insecure_credentials.cc \ - src/core/lib/security/credentials/plugin/plugin_credentials.cc \ - src/core/lib/security/credentials/tls/tls_utils.cc \ - src/core/lib/security/security_connector/fake/fake_security_connector.cc \ - src/core/lib/security/security_connector/insecure/insecure_security_connector.cc \ - src/core/lib/security/security_connector/load_system_roots_fallback.cc \ - src/core/lib/security/security_connector/load_system_roots_supported.cc \ - src/core/lib/security/security_connector/load_system_roots_windows.cc \ - src/core/lib/security/security_connector/security_connector.cc \ - src/core/lib/security/transport/client_auth_filter.cc \ - src/core/lib/security/transport/legacy_server_auth_filter.cc \ - src/core/lib/security/transport/secure_endpoint.cc \ - src/core/lib/security/transport/security_handshaker.cc \ - src/core/lib/security/transport/server_auth_filter.cc \ - src/core/lib/security/transport/tsi_error.cc \ - src/core/lib/security/util/json_util.cc \ - src/core/lib/slice/b64.cc \ - src/core/lib/slice/percent_encoding.cc \ - src/core/lib/slice/slice.cc \ - src/core/lib/slice/slice_buffer.cc \ - src/core/lib/slice/slice_refcount.cc \ - src/core/lib/slice/slice_string_helpers.cc \ - src/core/lib/surface/api_trace.cc \ - src/core/lib/surface/byte_buffer.cc \ - src/core/lib/surface/byte_buffer_reader.cc \ - src/core/lib/surface/call.cc \ - src/core/lib/surface/call_details.cc \ - src/core/lib/surface/call_log_batch.cc \ - src/core/lib/surface/channel.cc \ - src/core/lib/surface/channel_create.cc \ - src/core/lib/surface/channel_init.cc \ - src/core/lib/surface/channel_stack_type.cc \ - src/core/lib/surface/completion_queue.cc \ - src/core/lib/surface/completion_queue_factory.cc \ - src/core/lib/surface/event_string.cc \ - src/core/lib/surface/init.cc \ - src/core/lib/surface/init_internally.cc \ - src/core/lib/surface/lame_client.cc \ - src/core/lib/surface/legacy_channel.cc \ - src/core/lib/surface/metadata_array.cc \ - src/core/lib/surface/server.cc \ - src/core/lib/surface/validate_metadata.cc \ - src/core/lib/surface/version.cc \ - src/core/lib/surface/wait_for_cq_end_op.cc \ - src/core/lib/transport/batch_builder.cc \ - src/core/lib/transport/bdp_estimator.cc \ - src/core/lib/transport/call_factory.cc \ - src/core/lib/transport/call_filters.cc \ - src/core/lib/transport/call_final_info.cc \ - src/core/lib/transport/call_size_estimator.cc \ - src/core/lib/transport/call_spine.cc \ - src/core/lib/transport/connectivity_state.cc \ - src/core/lib/transport/error_utils.cc \ - src/core/lib/transport/handshaker.cc \ - src/core/lib/transport/handshaker_registry.cc \ - src/core/lib/transport/http_connect_handshaker.cc \ - src/core/lib/transport/message.cc \ - src/core/lib/transport/metadata.cc \ - src/core/lib/transport/metadata_batch.cc \ - src/core/lib/transport/metadata_info.cc \ - src/core/lib/transport/parsed_metadata.cc \ - src/core/lib/transport/status_conversion.cc \ - src/core/lib/transport/tcp_connect_handshaker.cc \ - src/core/lib/transport/timeout_encoding.cc \ - src/core/lib/transport/transport.cc \ - src/core/lib/transport/transport_op_string.cc \ - src/core/lib/uri/uri_parser.cc \ - src/core/load_balancing/address_filtering.cc \ - src/core/load_balancing/backend_metric_parser.cc \ - src/core/load_balancing/child_policy_handler.cc \ - src/core/load_balancing/endpoint_list.cc \ - src/core/load_balancing/grpclb/client_load_reporting_filter.cc \ - src/core/load_balancing/grpclb/grpclb.cc \ - src/core/load_balancing/grpclb/grpclb_balancer_addresses.cc \ - src/core/load_balancing/grpclb/grpclb_client_stats.cc \ - src/core/load_balancing/grpclb/load_balancer_api.cc \ - src/core/load_balancing/health_check_client.cc \ - src/core/load_balancing/lb_policy.cc \ - src/core/load_balancing/lb_policy_registry.cc \ - src/core/load_balancing/oob_backend_metric.cc \ - src/core/load_balancing/outlier_detection/outlier_detection.cc \ - src/core/load_balancing/pick_first/pick_first.cc \ - src/core/load_balancing/priority/priority.cc \ - src/core/load_balancing/rls/rls.cc \ - src/core/load_balancing/round_robin/round_robin.cc \ - src/core/load_balancing/weighted_round_robin/static_stride_scheduler.cc \ - src/core/load_balancing/weighted_round_robin/weighted_round_robin.cc \ - src/core/load_balancing/weighted_target/weighted_target.cc \ - src/core/plugin_registry/grpc_plugin_registry.cc \ - src/core/plugin_registry/grpc_plugin_registry_noextra.cc \ - src/core/resolver/binder/binder_resolver.cc \ - src/core/resolver/dns/c_ares/dns_resolver_ares.cc \ - src/core/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc \ - src/core/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc \ - src/core/resolver/dns/c_ares/grpc_ares_wrapper.cc \ - src/core/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc \ - src/core/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc \ - src/core/resolver/dns/dns_resolver_plugin.cc \ - src/core/resolver/dns/event_engine/event_engine_client_channel_resolver.cc \ - src/core/resolver/dns/event_engine/service_config_helper.cc \ - src/core/resolver/dns/native/dns_resolver.cc \ - src/core/resolver/endpoint_addresses.cc \ - src/core/resolver/fake/fake_resolver.cc \ - src/core/resolver/polling_resolver.cc \ - src/core/resolver/resolver.cc \ - src/core/resolver/resolver_registry.cc \ - src/core/resolver/sockaddr/sockaddr_resolver.cc \ - src/core/service_config/service_config_channel_arg_filter.cc \ - src/core/service_config/service_config_impl.cc \ - src/core/service_config/service_config_parser.cc \ - src/core/tsi/alts/handshaker/transport_security_common_api.cc \ - src/core/tsi/fake_transport_security.cc \ - src/core/tsi/local_transport_security.cc \ - src/core/tsi/transport_security.cc \ - src/core/tsi/transport_security_grpc.cc \ - third_party/upb/upb/message/accessors.c \ - third_party/upb/upb/mini_descriptor/build_enum.c \ - third_party/upb/upb/mini_descriptor/decode.c \ - third_party/upb/upb/mini_descriptor/internal/base92.c \ - third_party/upb/upb/mini_descriptor/internal/encode.c \ - third_party/upb/upb/mini_descriptor/link.c \ - third_party/upb/upb/wire/decode.c \ - third_party/upb/upb/wire/decode_fast.c \ - third_party/upb/upb/wire/encode.c \ - third_party/upb/upb/wire/eps_copy_input_stream.c \ - third_party/upb/upb/wire/reader.c \ - -PUBLIC_HEADERS_C += \ - include/grpc/byte_buffer.h \ - include/grpc/byte_buffer_reader.h \ - include/grpc/census.h \ - include/grpc/compression.h \ - include/grpc/event_engine/endpoint_config.h \ - include/grpc/event_engine/event_engine.h \ - include/grpc/event_engine/extensible.h \ - include/grpc/event_engine/internal/memory_allocator_impl.h \ - include/grpc/event_engine/internal/slice_cast.h \ - include/grpc/event_engine/memory_allocator.h \ - include/grpc/event_engine/memory_request.h \ - include/grpc/event_engine/port.h \ - include/grpc/event_engine/slice.h \ - include/grpc/event_engine/slice_buffer.h \ - include/grpc/fork.h \ - include/grpc/grpc.h \ - include/grpc/grpc_audit_logging.h \ - include/grpc/grpc_crl_provider.h \ - include/grpc/grpc_posix.h \ - include/grpc/grpc_security.h \ - include/grpc/grpc_security_constants.h \ - include/grpc/impl/call.h \ - include/grpc/impl/channel_arg_names.h \ - include/grpc/impl/codegen/atm.h \ - include/grpc/impl/codegen/atm_gcc_atomic.h \ - include/grpc/impl/codegen/atm_gcc_sync.h \ - include/grpc/impl/codegen/atm_windows.h \ - include/grpc/impl/codegen/byte_buffer.h \ - include/grpc/impl/codegen/byte_buffer_reader.h \ - include/grpc/impl/codegen/compression_types.h \ - include/grpc/impl/codegen/connectivity_state.h \ - include/grpc/impl/codegen/fork.h \ - include/grpc/impl/codegen/gpr_types.h \ - include/grpc/impl/codegen/grpc_types.h \ - include/grpc/impl/codegen/log.h \ - include/grpc/impl/codegen/port_platform.h \ - include/grpc/impl/codegen/propagation_bits.h \ - include/grpc/impl/codegen/slice.h \ - include/grpc/impl/codegen/status.h \ - include/grpc/impl/codegen/sync.h \ - include/grpc/impl/codegen/sync_abseil.h \ - include/grpc/impl/codegen/sync_custom.h \ - include/grpc/impl/codegen/sync_generic.h \ - include/grpc/impl/codegen/sync_posix.h \ - include/grpc/impl/codegen/sync_windows.h \ - include/grpc/impl/compression_types.h \ - include/grpc/impl/connectivity_state.h \ - include/grpc/impl/grpc_types.h \ - include/grpc/impl/propagation_bits.h \ - include/grpc/impl/slice_type.h \ - include/grpc/load_reporting.h \ - include/grpc/slice.h \ - include/grpc/slice_buffer.h \ - include/grpc/status.h \ - include/grpc/support/alloc.h \ - include/grpc/support/atm_gcc_atomic.h \ - include/grpc/support/atm_gcc_sync.h \ - include/grpc/support/atm_windows.h \ - include/grpc/support/cpu.h \ - include/grpc/support/json.h \ - include/grpc/support/log.h \ - include/grpc/support/log_windows.h \ - include/grpc/support/port_platform.h \ - include/grpc/support/string_util.h \ - include/grpc/support/sync.h \ - include/grpc/support/sync_abseil.h \ - include/grpc/support/sync_custom.h \ - include/grpc/support/sync_generic.h \ - include/grpc/support/sync_posix.h \ - include/grpc/support/sync_windows.h \ - include/grpc/support/thd_id.h \ - include/grpc/support/time.h \ - include/grpc/support/workaround_list.h \ - -LIBGRPC_UNSECURE_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBGRPC_UNSECURE_SRC)))) - - -# static library for "grpc_unsecure" -$(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a: $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libgpr.a $(GRPC_ABSEIL_DEP) $(LIBDIR)/$(CONFIG)/libcares.a $(ZLIB_DEP) $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(LIBGRPC_UNSECURE_OBJS) $(LIBADDRESS_SORTING_OBJS) $(LIBGPR_OBJS) $(LIBGRPC_ABSEIL_OBJS) $(LIBCARES_OBJS) $(ZLIB_MERGE_OBJS) $(LIBUTF8_RANGE_LIB_OBJS) $(LIBUPB_MESSAGE_LIB_OBJS) $(LIBUPB_MEM_LIB_OBJS) $(LIBUPB_BASE_LIB_OBJS) - $(E) "[AR] Creating $@" - $(Q) mkdir -p `dirname $@` - $(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a - $(Q) $(AR) $(ARFLAGS) $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a $(LIBGRPC_UNSECURE_OBJS) $(LIBADDRESS_SORTING_OBJS) $(LIBGPR_OBJS) $(LIBGRPC_ABSEIL_OBJS) $(LIBCARES_OBJS) $(ZLIB_MERGE_OBJS) $(LIBUTF8_RANGE_LIB_OBJS) $(LIBUPB_MESSAGE_LIB_OBJS) $(LIBUPB_MEM_LIB_OBJS) $(LIBUPB_BASE_LIB_OBJS) -ifeq ($(SYSTEM),Darwin) - $(Q) $(RANLIB) $(RANLIBFLAGS) $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a -endif - -# shared library for "grpc_unsecure" -ifeq ($(SYSTEM),MINGW32) -$(LIBDIR)/$(CONFIG)/grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBGRPC_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libgpr.a $(GRPC_ABSEIL_DEP) $(LIBDIR)/$(CONFIG)/libcares.a $(ZLIB_DEP) $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a - $(E) "[LD] Linking $@" - $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,--output-def=$(LIBDIR)/$(CONFIG)/grpc_unsecure$(SHARED_VERSION_CORE).def -Wl,--out-implib=$(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE)-dll.a -o $(LIBDIR)/$(CONFIG)/grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libgpr.a $(GRPC_ABSEIL_MERGE_LIBS) $(LIBDIR)/$(CONFIG)/libcares.a $(ZLIB_MERGE_LIBS) $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(LDLIBS) -else -$(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBGRPC_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libgpr.a $(GRPC_ABSEIL_DEP) $(LIBDIR)/$(CONFIG)/libcares.a $(ZLIB_DEP) $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a - $(E) "[LD] Linking $@" - $(Q) mkdir -p `dirname $@` -ifeq ($(SYSTEM),Darwin) - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libgpr.a $(GRPC_ABSEIL_MERGE_LIBS) $(LIBDIR)/$(CONFIG)/libcares.a $(ZLIB_MERGE_LIBS) $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(LDLIBS) -else - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgrpc_unsecure.so.39 -o $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libgpr.a $(GRPC_ABSEIL_MERGE_LIBS) $(LIBDIR)/$(CONFIG)/libcares.a $(ZLIB_MERGE_LIBS) $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(LDLIBS) - $(Q) ln -sf $(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).so.39 - $(Q) ln -sf $(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).so -endif -endif - -ifneq ($(NO_DEPS),true) --include $(LIBGRPC_UNSECURE_OBJS:.o=.dep) -endif -# end of build recipe for library "grpc_unsecure" - - -# start of build recipe for library "re2" (generated by makelib(lib) template function) -# deps: [] -# transitive_deps: [] -LIBRE2_SRC = \ - third_party/re2/re2/bitstate.cc \ - third_party/re2/re2/compile.cc \ - third_party/re2/re2/dfa.cc \ - third_party/re2/re2/filtered_re2.cc \ - third_party/re2/re2/mimics_pcre.cc \ - third_party/re2/re2/nfa.cc \ - third_party/re2/re2/onepass.cc \ - third_party/re2/re2/parse.cc \ - third_party/re2/re2/perl_groups.cc \ - third_party/re2/re2/prefilter.cc \ - third_party/re2/re2/prefilter_tree.cc \ - third_party/re2/re2/prog.cc \ - third_party/re2/re2/re2.cc \ - third_party/re2/re2/regexp.cc \ - third_party/re2/re2/set.cc \ - third_party/re2/re2/simplify.cc \ - third_party/re2/re2/stringpiece.cc \ - third_party/re2/re2/tostring.cc \ - third_party/re2/re2/unicode_casefold.cc \ - third_party/re2/re2/unicode_groups.cc \ - third_party/re2/util/rune.cc \ - third_party/re2/util/strutil.cc \ - -PUBLIC_HEADERS_C += \ - -LIBRE2_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBRE2_SRC)))) - - -# static library for "re2" -$(LIBDIR)/$(CONFIG)/libre2.a: $(LIBRE2_OBJS) - $(E) "[AR] Creating $@" - $(Q) mkdir -p `dirname $@` - $(Q) rm -f $(LIBDIR)/$(CONFIG)/libre2.a - $(Q) $(AR) $(ARFLAGS) $(LIBDIR)/$(CONFIG)/libre2.a $(LIBRE2_OBJS) -ifeq ($(SYSTEM),Darwin) - $(Q) $(RANLIB) $(RANLIBFLAGS) $(LIBDIR)/$(CONFIG)/libre2.a -endif - -# shared library for "re2" -ifeq ($(SYSTEM),MINGW32) -$(LIBDIR)/$(CONFIG)/re2$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBRE2_OBJS) - $(E) "[LD] Linking $@" - $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,--output-def=$(LIBDIR)/$(CONFIG)/re2$(SHARED_VERSION_CORE).def -Wl,--out-implib=$(LIBDIR)/$(CONFIG)/libre2$(SHARED_VERSION_CORE)-dll.a -o $(LIBDIR)/$(CONFIG)/re2$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBRE2_OBJS) $(LDLIBS) -else -$(LIBDIR)/$(CONFIG)/libre2$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBRE2_OBJS) - $(E) "[LD] Linking $@" - $(Q) mkdir -p `dirname $@` -ifeq ($(SYSTEM),Darwin) - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)re2$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libre2$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBRE2_OBJS) $(LDLIBS) -else - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libre2.so.39 -o $(LIBDIR)/$(CONFIG)/libre2$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBRE2_OBJS) $(LDLIBS) - $(Q) ln -sf $(SHARED_PREFIX)re2$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libre2$(SHARED_VERSION_CORE).so.39 - $(Q) ln -sf $(SHARED_PREFIX)re2$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libre2$(SHARED_VERSION_CORE).so -endif -endif - -ifneq ($(NO_DEPS),true) --include $(LIBRE2_OBJS:.o=.dep) -endif -# end of build recipe for library "re2" - - -# start of build recipe for library "upb_base_lib" (generated by makelib(lib) template function) -# deps: [] -# transitive_deps: [] -LIBUPB_BASE_LIB_SRC = \ - third_party/upb/upb/base/status.c \ - -PUBLIC_HEADERS_C += \ - -LIBUPB_BASE_LIB_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBUPB_BASE_LIB_SRC)))) - - -# static library for "upb_base_lib" -$(LIBDIR)/$(CONFIG)/libupb_base_lib.a: $(LIBUPB_BASE_LIB_OBJS) - $(E) "[AR] Creating $@" - $(Q) mkdir -p `dirname $@` - $(Q) rm -f $(LIBDIR)/$(CONFIG)/libupb_base_lib.a - $(Q) $(AR) $(ARFLAGS) $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(LIBUPB_BASE_LIB_OBJS) -ifeq ($(SYSTEM),Darwin) - $(Q) $(RANLIB) $(RANLIBFLAGS) $(LIBDIR)/$(CONFIG)/libupb_base_lib.a -endif - -# shared library for "upb_base_lib" -ifeq ($(SYSTEM),MINGW32) -$(LIBDIR)/$(CONFIG)/upb_base_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBUPB_BASE_LIB_OBJS) - $(E) "[LD] Linking $@" - $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,--output-def=$(LIBDIR)/$(CONFIG)/upb_base_lib$(SHARED_VERSION_CORE).def -Wl,--out-implib=$(LIBDIR)/$(CONFIG)/libupb_base_lib$(SHARED_VERSION_CORE)-dll.a -o $(LIBDIR)/$(CONFIG)/upb_base_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUPB_BASE_LIB_OBJS) $(LDLIBS) -else -$(LIBDIR)/$(CONFIG)/libupb_base_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBUPB_BASE_LIB_OBJS) - $(E) "[LD] Linking $@" - $(Q) mkdir -p `dirname $@` -ifeq ($(SYSTEM),Darwin) - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)upb_base_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libupb_base_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUPB_BASE_LIB_OBJS) $(LDLIBS) -else - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libupb_base_lib.so.39 -o $(LIBDIR)/$(CONFIG)/libupb_base_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUPB_BASE_LIB_OBJS) $(LDLIBS) - $(Q) ln -sf $(SHARED_PREFIX)upb_base_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libupb_base_lib$(SHARED_VERSION_CORE).so.39 - $(Q) ln -sf $(SHARED_PREFIX)upb_base_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libupb_base_lib$(SHARED_VERSION_CORE).so -endif -endif - -ifneq ($(NO_DEPS),true) --include $(LIBUPB_BASE_LIB_OBJS:.o=.dep) -endif -# end of build recipe for library "upb_base_lib" - - -# start of build recipe for library "upb_json_lib" (generated by makelib(lib) template function) -# deps: ['upb_message_lib', 'utf8_range_lib'] -# transitive_deps: ['utf8_range_lib', 'upb_message_lib', 'upb_mem_lib', 'upb_base_lib'] -LIBUPB_JSON_LIB_SRC = \ - src/core/ext/upb-gen/google/protobuf/descriptor.upb_minitable.c \ - third_party/upb/upb/json/decode.c \ - third_party/upb/upb/json/encode.c \ - third_party/upb/upb/lex/atoi.c \ - third_party/upb/upb/lex/round_trip.c \ - third_party/upb/upb/lex/strtod.c \ - third_party/upb/upb/lex/unicode.c \ - third_party/upb/upb/message/accessors.c \ - third_party/upb/upb/mini_descriptor/build_enum.c \ - third_party/upb/upb/mini_descriptor/decode.c \ - third_party/upb/upb/mini_descriptor/internal/base92.c \ - third_party/upb/upb/mini_descriptor/internal/encode.c \ - third_party/upb/upb/mini_descriptor/link.c \ - third_party/upb/upb/reflection/def_pool.c \ - third_party/upb/upb/reflection/def_type.c \ - third_party/upb/upb/reflection/desc_state.c \ - third_party/upb/upb/reflection/enum_def.c \ - third_party/upb/upb/reflection/enum_reserved_range.c \ - third_party/upb/upb/reflection/enum_value_def.c \ - third_party/upb/upb/reflection/extension_range.c \ - third_party/upb/upb/reflection/field_def.c \ - third_party/upb/upb/reflection/file_def.c \ - third_party/upb/upb/reflection/internal/def_builder.c \ - third_party/upb/upb/reflection/internal/strdup2.c \ - third_party/upb/upb/reflection/message.c \ - third_party/upb/upb/reflection/message_def.c \ - third_party/upb/upb/reflection/message_reserved_range.c \ - third_party/upb/upb/reflection/method_def.c \ - third_party/upb/upb/reflection/oneof_def.c \ - third_party/upb/upb/reflection/service_def.c \ - third_party/upb/upb/wire/decode.c \ - third_party/upb/upb/wire/decode_fast.c \ - third_party/upb/upb/wire/encode.c \ - third_party/upb/upb/wire/eps_copy_input_stream.c \ - third_party/upb/upb/wire/reader.c \ - -PUBLIC_HEADERS_C += \ - -LIBUPB_JSON_LIB_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBUPB_JSON_LIB_SRC)))) - - -# static library for "upb_json_lib" -$(LIBDIR)/$(CONFIG)/libupb_json_lib.a: $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(LIBUPB_JSON_LIB_OBJS) $(LIBUTF8_RANGE_LIB_OBJS) $(LIBUPB_MESSAGE_LIB_OBJS) $(LIBUPB_MEM_LIB_OBJS) $(LIBUPB_BASE_LIB_OBJS) - $(E) "[AR] Creating $@" - $(Q) mkdir -p `dirname $@` - $(Q) rm -f $(LIBDIR)/$(CONFIG)/libupb_json_lib.a - $(Q) $(AR) $(ARFLAGS) $(LIBDIR)/$(CONFIG)/libupb_json_lib.a $(LIBUPB_JSON_LIB_OBJS) $(LIBUTF8_RANGE_LIB_OBJS) $(LIBUPB_MESSAGE_LIB_OBJS) $(LIBUPB_MEM_LIB_OBJS) $(LIBUPB_BASE_LIB_OBJS) -ifeq ($(SYSTEM),Darwin) - $(Q) $(RANLIB) $(RANLIBFLAGS) $(LIBDIR)/$(CONFIG)/libupb_json_lib.a -endif - -# shared library for "upb_json_lib" -ifeq ($(SYSTEM),MINGW32) -$(LIBDIR)/$(CONFIG)/upb_json_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBUPB_JSON_LIB_OBJS) $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a - $(E) "[LD] Linking $@" - $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,--output-def=$(LIBDIR)/$(CONFIG)/upb_json_lib$(SHARED_VERSION_CORE).def -Wl,--out-implib=$(LIBDIR)/$(CONFIG)/libupb_json_lib$(SHARED_VERSION_CORE)-dll.a -o $(LIBDIR)/$(CONFIG)/upb_json_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUPB_JSON_LIB_OBJS) $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(LDLIBS) -else -$(LIBDIR)/$(CONFIG)/libupb_json_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBUPB_JSON_LIB_OBJS) $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a - $(E) "[LD] Linking $@" - $(Q) mkdir -p `dirname $@` -ifeq ($(SYSTEM),Darwin) - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)upb_json_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libupb_json_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUPB_JSON_LIB_OBJS) $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(LDLIBS) -else - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libupb_json_lib.so.39 -o $(LIBDIR)/$(CONFIG)/libupb_json_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUPB_JSON_LIB_OBJS) $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(LDLIBS) - $(Q) ln -sf $(SHARED_PREFIX)upb_json_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libupb_json_lib$(SHARED_VERSION_CORE).so.39 - $(Q) ln -sf $(SHARED_PREFIX)upb_json_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libupb_json_lib$(SHARED_VERSION_CORE).so -endif -endif - -ifneq ($(NO_DEPS),true) --include $(LIBUPB_JSON_LIB_OBJS:.o=.dep) -endif -# end of build recipe for library "upb_json_lib" - - -# start of build recipe for library "upb_mem_lib" (generated by makelib(lib) template function) -# deps: [] -# transitive_deps: [] -LIBUPB_MEM_LIB_SRC = \ - third_party/upb/upb/mem/alloc.c \ - third_party/upb/upb/mem/arena.c \ - -PUBLIC_HEADERS_C += \ - -LIBUPB_MEM_LIB_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBUPB_MEM_LIB_SRC)))) - - -# static library for "upb_mem_lib" -$(LIBDIR)/$(CONFIG)/libupb_mem_lib.a: $(LIBUPB_MEM_LIB_OBJS) - $(E) "[AR] Creating $@" - $(Q) mkdir -p `dirname $@` - $(Q) rm -f $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a - $(Q) $(AR) $(ARFLAGS) $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBUPB_MEM_LIB_OBJS) -ifeq ($(SYSTEM),Darwin) - $(Q) $(RANLIB) $(RANLIBFLAGS) $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a -endif - -# shared library for "upb_mem_lib" -ifeq ($(SYSTEM),MINGW32) -$(LIBDIR)/$(CONFIG)/upb_mem_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBUPB_MEM_LIB_OBJS) - $(E) "[LD] Linking $@" - $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,--output-def=$(LIBDIR)/$(CONFIG)/upb_mem_lib$(SHARED_VERSION_CORE).def -Wl,--out-implib=$(LIBDIR)/$(CONFIG)/libupb_mem_lib$(SHARED_VERSION_CORE)-dll.a -o $(LIBDIR)/$(CONFIG)/upb_mem_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUPB_MEM_LIB_OBJS) $(LDLIBS) -else -$(LIBDIR)/$(CONFIG)/libupb_mem_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBUPB_MEM_LIB_OBJS) - $(E) "[LD] Linking $@" - $(Q) mkdir -p `dirname $@` -ifeq ($(SYSTEM),Darwin) - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)upb_mem_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libupb_mem_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUPB_MEM_LIB_OBJS) $(LDLIBS) -else - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libupb_mem_lib.so.39 -o $(LIBDIR)/$(CONFIG)/libupb_mem_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUPB_MEM_LIB_OBJS) $(LDLIBS) - $(Q) ln -sf $(SHARED_PREFIX)upb_mem_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libupb_mem_lib$(SHARED_VERSION_CORE).so.39 - $(Q) ln -sf $(SHARED_PREFIX)upb_mem_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libupb_mem_lib$(SHARED_VERSION_CORE).so -endif -endif - -ifneq ($(NO_DEPS),true) --include $(LIBUPB_MEM_LIB_OBJS:.o=.dep) -endif -# end of build recipe for library "upb_mem_lib" - - -# start of build recipe for library "upb_message_lib" (generated by makelib(lib) template function) -# deps: ['upb_base_lib', 'upb_mem_lib'] -# transitive_deps: ['upb_mem_lib', 'upb_base_lib'] -LIBUPB_MESSAGE_LIB_SRC = \ - third_party/upb/upb/hash/common.c \ - third_party/upb/upb/message/array.c \ - third_party/upb/upb/message/map.c \ - third_party/upb/upb/message/map_sorter.c \ - third_party/upb/upb/message/message.c \ - third_party/upb/upb/mini_table/extension_registry.c \ - third_party/upb/upb/mini_table/internal/message.c \ - third_party/upb/upb/mini_table/message.c \ - -PUBLIC_HEADERS_C += \ - -LIBUPB_MESSAGE_LIB_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBUPB_MESSAGE_LIB_SRC)))) - - -# static library for "upb_message_lib" -$(LIBDIR)/$(CONFIG)/libupb_message_lib.a: $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(LIBUPB_MESSAGE_LIB_OBJS) $(LIBUPB_MEM_LIB_OBJS) $(LIBUPB_BASE_LIB_OBJS) - $(E) "[AR] Creating $@" - $(Q) mkdir -p `dirname $@` - $(Q) rm -f $(LIBDIR)/$(CONFIG)/libupb_message_lib.a - $(Q) $(AR) $(ARFLAGS) $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBUPB_MESSAGE_LIB_OBJS) $(LIBUPB_MEM_LIB_OBJS) $(LIBUPB_BASE_LIB_OBJS) -ifeq ($(SYSTEM),Darwin) - $(Q) $(RANLIB) $(RANLIBFLAGS) $(LIBDIR)/$(CONFIG)/libupb_message_lib.a -endif - -# shared library for "upb_message_lib" -ifeq ($(SYSTEM),MINGW32) -$(LIBDIR)/$(CONFIG)/upb_message_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBUPB_MESSAGE_LIB_OBJS) $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a - $(E) "[LD] Linking $@" - $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,--output-def=$(LIBDIR)/$(CONFIG)/upb_message_lib$(SHARED_VERSION_CORE).def -Wl,--out-implib=$(LIBDIR)/$(CONFIG)/libupb_message_lib$(SHARED_VERSION_CORE)-dll.a -o $(LIBDIR)/$(CONFIG)/upb_message_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUPB_MESSAGE_LIB_OBJS) $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(LDLIBS) -else -$(LIBDIR)/$(CONFIG)/libupb_message_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBUPB_MESSAGE_LIB_OBJS) $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a - $(E) "[LD] Linking $@" - $(Q) mkdir -p `dirname $@` -ifeq ($(SYSTEM),Darwin) - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)upb_message_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libupb_message_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUPB_MESSAGE_LIB_OBJS) $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(LDLIBS) -else - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libupb_message_lib.so.39 -o $(LIBDIR)/$(CONFIG)/libupb_message_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUPB_MESSAGE_LIB_OBJS) $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(LDLIBS) - $(Q) ln -sf $(SHARED_PREFIX)upb_message_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libupb_message_lib$(SHARED_VERSION_CORE).so.39 - $(Q) ln -sf $(SHARED_PREFIX)upb_message_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libupb_message_lib$(SHARED_VERSION_CORE).so -endif -endif - -ifneq ($(NO_DEPS),true) --include $(LIBUPB_MESSAGE_LIB_OBJS:.o=.dep) -endif -# end of build recipe for library "upb_message_lib" - - -# start of build recipe for library "upb_textformat_lib" (generated by makelib(lib) template function) -# deps: ['upb_message_lib', 'utf8_range_lib'] -# transitive_deps: ['utf8_range_lib', 'upb_message_lib', 'upb_mem_lib', 'upb_base_lib'] -LIBUPB_TEXTFORMAT_LIB_SRC = \ - src/core/ext/upb-gen/google/protobuf/descriptor.upb_minitable.c \ - third_party/upb/upb/lex/atoi.c \ - third_party/upb/upb/lex/round_trip.c \ - third_party/upb/upb/lex/strtod.c \ - third_party/upb/upb/lex/unicode.c \ - third_party/upb/upb/message/accessors.c \ - third_party/upb/upb/mini_descriptor/build_enum.c \ - third_party/upb/upb/mini_descriptor/decode.c \ - third_party/upb/upb/mini_descriptor/internal/base92.c \ - third_party/upb/upb/mini_descriptor/internal/encode.c \ - third_party/upb/upb/mini_descriptor/link.c \ - third_party/upb/upb/reflection/def_pool.c \ - third_party/upb/upb/reflection/def_type.c \ - third_party/upb/upb/reflection/desc_state.c \ - third_party/upb/upb/reflection/enum_def.c \ - third_party/upb/upb/reflection/enum_reserved_range.c \ - third_party/upb/upb/reflection/enum_value_def.c \ - third_party/upb/upb/reflection/extension_range.c \ - third_party/upb/upb/reflection/field_def.c \ - third_party/upb/upb/reflection/file_def.c \ - third_party/upb/upb/reflection/internal/def_builder.c \ - third_party/upb/upb/reflection/internal/strdup2.c \ - third_party/upb/upb/reflection/message.c \ - third_party/upb/upb/reflection/message_def.c \ - third_party/upb/upb/reflection/message_reserved_range.c \ - third_party/upb/upb/reflection/method_def.c \ - third_party/upb/upb/reflection/oneof_def.c \ - third_party/upb/upb/reflection/service_def.c \ - third_party/upb/upb/text/encode.c \ - third_party/upb/upb/wire/decode.c \ - third_party/upb/upb/wire/decode_fast.c \ - third_party/upb/upb/wire/encode.c \ - third_party/upb/upb/wire/eps_copy_input_stream.c \ - third_party/upb/upb/wire/reader.c \ - -PUBLIC_HEADERS_C += \ - -LIBUPB_TEXTFORMAT_LIB_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBUPB_TEXTFORMAT_LIB_SRC)))) - - -# static library for "upb_textformat_lib" -$(LIBDIR)/$(CONFIG)/libupb_textformat_lib.a: $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(LIBUPB_TEXTFORMAT_LIB_OBJS) $(LIBUTF8_RANGE_LIB_OBJS) $(LIBUPB_MESSAGE_LIB_OBJS) $(LIBUPB_MEM_LIB_OBJS) $(LIBUPB_BASE_LIB_OBJS) - $(E) "[AR] Creating $@" - $(Q) mkdir -p `dirname $@` - $(Q) rm -f $(LIBDIR)/$(CONFIG)/libupb_textformat_lib.a - $(Q) $(AR) $(ARFLAGS) $(LIBDIR)/$(CONFIG)/libupb_textformat_lib.a $(LIBUPB_TEXTFORMAT_LIB_OBJS) $(LIBUTF8_RANGE_LIB_OBJS) $(LIBUPB_MESSAGE_LIB_OBJS) $(LIBUPB_MEM_LIB_OBJS) $(LIBUPB_BASE_LIB_OBJS) -ifeq ($(SYSTEM),Darwin) - $(Q) $(RANLIB) $(RANLIBFLAGS) $(LIBDIR)/$(CONFIG)/libupb_textformat_lib.a -endif - -# shared library for "upb_textformat_lib" -ifeq ($(SYSTEM),MINGW32) -$(LIBDIR)/$(CONFIG)/upb_textformat_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBUPB_TEXTFORMAT_LIB_OBJS) $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a - $(E) "[LD] Linking $@" - $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,--output-def=$(LIBDIR)/$(CONFIG)/upb_textformat_lib$(SHARED_VERSION_CORE).def -Wl,--out-implib=$(LIBDIR)/$(CONFIG)/libupb_textformat_lib$(SHARED_VERSION_CORE)-dll.a -o $(LIBDIR)/$(CONFIG)/upb_textformat_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUPB_TEXTFORMAT_LIB_OBJS) $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(LDLIBS) -else -$(LIBDIR)/$(CONFIG)/libupb_textformat_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBUPB_TEXTFORMAT_LIB_OBJS) $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a - $(E) "[LD] Linking $@" - $(Q) mkdir -p `dirname $@` -ifeq ($(SYSTEM),Darwin) - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)upb_textformat_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libupb_textformat_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUPB_TEXTFORMAT_LIB_OBJS) $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(LDLIBS) -else - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libupb_textformat_lib.so.39 -o $(LIBDIR)/$(CONFIG)/libupb_textformat_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUPB_TEXTFORMAT_LIB_OBJS) $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(LDLIBS) - $(Q) ln -sf $(SHARED_PREFIX)upb_textformat_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libupb_textformat_lib$(SHARED_VERSION_CORE).so.39 - $(Q) ln -sf $(SHARED_PREFIX)upb_textformat_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libupb_textformat_lib$(SHARED_VERSION_CORE).so -endif -endif - -ifneq ($(NO_DEPS),true) --include $(LIBUPB_TEXTFORMAT_LIB_OBJS:.o=.dep) -endif -# end of build recipe for library "upb_textformat_lib" - - -# start of build recipe for library "utf8_range_lib" (generated by makelib(lib) template function) -# deps: [] -# transitive_deps: [] -LIBUTF8_RANGE_LIB_SRC = \ - third_party/utf8_range/naive.c \ - third_party/utf8_range/range2-neon.c \ - third_party/utf8_range/range2-sse.c \ - -PUBLIC_HEADERS_C += \ - -LIBUTF8_RANGE_LIB_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBUTF8_RANGE_LIB_SRC)))) - - -# static library for "utf8_range_lib" -$(LIBDIR)/$(CONFIG)/libutf8_range_lib.a: $(LIBUTF8_RANGE_LIB_OBJS) - $(E) "[AR] Creating $@" - $(Q) mkdir -p `dirname $@` - $(Q) rm -f $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a - $(Q) $(AR) $(ARFLAGS) $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a $(LIBUTF8_RANGE_LIB_OBJS) -ifeq ($(SYSTEM),Darwin) - $(Q) $(RANLIB) $(RANLIBFLAGS) $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a -endif - -# shared library for "utf8_range_lib" -ifeq ($(SYSTEM),MINGW32) -$(LIBDIR)/$(CONFIG)/utf8_range_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBUTF8_RANGE_LIB_OBJS) - $(E) "[LD] Linking $@" - $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,--output-def=$(LIBDIR)/$(CONFIG)/utf8_range_lib$(SHARED_VERSION_CORE).def -Wl,--out-implib=$(LIBDIR)/$(CONFIG)/libutf8_range_lib$(SHARED_VERSION_CORE)-dll.a -o $(LIBDIR)/$(CONFIG)/utf8_range_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUTF8_RANGE_LIB_OBJS) $(LDLIBS) -else -$(LIBDIR)/$(CONFIG)/libutf8_range_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBUTF8_RANGE_LIB_OBJS) - $(E) "[LD] Linking $@" - $(Q) mkdir -p `dirname $@` -ifeq ($(SYSTEM),Darwin) - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)utf8_range_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libutf8_range_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUTF8_RANGE_LIB_OBJS) $(LDLIBS) -else - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libutf8_range_lib.so.39 -o $(LIBDIR)/$(CONFIG)/libutf8_range_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUTF8_RANGE_LIB_OBJS) $(LDLIBS) - $(Q) ln -sf $(SHARED_PREFIX)utf8_range_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libutf8_range_lib$(SHARED_VERSION_CORE).so.39 - $(Q) ln -sf $(SHARED_PREFIX)utf8_range_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libutf8_range_lib$(SHARED_VERSION_CORE).so -endif -endif - -ifneq ($(NO_DEPS),true) --include $(LIBUTF8_RANGE_LIB_OBJS:.o=.dep) -endif -# end of build recipe for library "utf8_range_lib" - - -# start of build recipe for library "z" (generated by makelib(lib) template function) -# deps: [] -# transitive_deps: [] -LIBZ_SRC = \ - third_party/zlib/adler32.c \ - third_party/zlib/compress.c \ - third_party/zlib/crc32.c \ - third_party/zlib/deflate.c \ - third_party/zlib/infback.c \ - third_party/zlib/inffast.c \ - third_party/zlib/inflate.c \ - third_party/zlib/inftrees.c \ - third_party/zlib/trees.c \ - third_party/zlib/uncompr.c \ - third_party/zlib/zutil.c \ - -PUBLIC_HEADERS_C += \ - -LIBZ_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBZ_SRC)))) - -$(LIBZ_OBJS): CFLAGS += -fvisibility=hidden -$(LIBZ_OBJS): CPPFLAGS += -DHAVE_UNISTD_H - -# static library for "z" -$(LIBDIR)/$(CONFIG)/libz.a: $(ZLIB_MERGE_OBJS) - $(E) "[AR] Creating $@" - $(Q) mkdir -p `dirname $@` - $(Q) rm -f $(LIBDIR)/$(CONFIG)/libz.a - $(Q) $(AR) $(ARFLAGS) $(LIBDIR)/$(CONFIG)/libz.a $(LIBZ_OBJS) -ifeq ($(SYSTEM),Darwin) - $(Q) $(RANLIB) $(RANLIBFLAGS) $(LIBDIR)/$(CONFIG)/libz.a -endif - -# shared library for "z" - -ifneq ($(NO_DEPS),true) --include $(LIBZ_OBJS:.o=.dep) -endif -# end of build recipe for library "z" - - # start of build recipe for library "boringssl" (generated by makelib(lib) template function) # deps: [] # transitive_deps: [] @@ -3179,66 +2141,66 @@ endif # deps: [] # transitive_deps: [] LIBCARES_SRC = \ - third_party/cares/cares/src/lib/ares__read_line.c \ - third_party/cares/cares/src/lib/ares__get_hostent.c \ + third_party/cares/cares/src/lib/ares__addrinfo2hostent.c \ + third_party/cares/cares/src/lib/ares__addrinfo_localhost.c \ third_party/cares/cares/src/lib/ares__close_sockets.c \ + third_party/cares/cares/src/lib/ares__get_hostent.c \ + third_party/cares/cares/src/lib/ares__parse_into_addrinfo.c \ + third_party/cares/cares/src/lib/ares__read_line.c \ + third_party/cares/cares/src/lib/ares__readaddrinfo.c \ + third_party/cares/cares/src/lib/ares__sortaddrinfo.c \ third_party/cares/cares/src/lib/ares__timeval.c \ - third_party/cares/cares/src/lib/ares_gethostbyaddr.c \ - third_party/cares/cares/src/lib/ares_getenv.c \ - third_party/cares/cares/src/lib/ares_free_string.c \ - third_party/cares/cares/src/lib/ares_free_hostent.c \ - third_party/cares/cares/src/lib/ares_fds.c \ - third_party/cares/cares/src/lib/ares_expand_string.c \ - third_party/cares/cares/src/lib/ares_create_query.c \ - third_party/cares/cares/src/lib/ares_cancel.c \ third_party/cares/cares/src/lib/ares_android.c \ - third_party/cares/cares/src/lib/ares_parse_txt_reply.c \ - third_party/cares/cares/src/lib/ares_parse_srv_reply.c \ - third_party/cares/cares/src/lib/ares_parse_soa_reply.c \ - third_party/cares/cares/src/lib/ares_parse_ptr_reply.c \ - third_party/cares/cares/src/lib/ares_parse_ns_reply.c \ - third_party/cares/cares/src/lib/ares_parse_naptr_reply.c \ - third_party/cares/cares/src/lib/ares_parse_mx_reply.c \ - third_party/cares/cares/src/lib/ares_parse_caa_reply.c \ - third_party/cares/cares/src/lib/ares_options.c \ - third_party/cares/cares/src/lib/ares_nowarn.c \ - third_party/cares/cares/src/lib/ares_mkquery.c \ - third_party/cares/cares/src/lib/ares_llist.c \ - third_party/cares/cares/src/lib/ares_getsock.c \ + third_party/cares/cares/src/lib/ares_cancel.c \ + third_party/cares/cares/src/lib/ares_create_query.c \ + third_party/cares/cares/src/lib/ares_data.c \ + third_party/cares/cares/src/lib/ares_destroy.c \ + third_party/cares/cares/src/lib/ares_expand_name.c \ + third_party/cares/cares/src/lib/ares_expand_string.c \ + third_party/cares/cares/src/lib/ares_fds.c \ + third_party/cares/cares/src/lib/ares_free_hostent.c \ + third_party/cares/cares/src/lib/ares_free_string.c \ + third_party/cares/cares/src/lib/ares_freeaddrinfo.c \ + third_party/cares/cares/src/lib/ares_getaddrinfo.c \ + third_party/cares/cares/src/lib/ares_getenv.c \ + third_party/cares/cares/src/lib/ares_gethostbyaddr.c \ + third_party/cares/cares/src/lib/ares_gethostbyname.c \ third_party/cares/cares/src/lib/ares_getnameinfo.c \ - third_party/cares/cares/src/lib/bitncmp.c \ - third_party/cares/cares/src/lib/ares_writev.c \ - third_party/cares/cares/src/lib/ares_version.c \ - third_party/cares/cares/src/lib/ares_timeout.c \ - third_party/cares/cares/src/lib/ares_strerror.c \ - third_party/cares/cares/src/lib/ares_strcasecmp.c \ - third_party/cares/cares/src/lib/ares_search.c \ - third_party/cares/cares/src/lib/ares_platform.c \ - third_party/cares/cares/src/lib/windows_port.c \ - third_party/cares/cares/src/lib/inet_ntop.c \ - third_party/cares/cares/src/lib/ares__sortaddrinfo.c \ - third_party/cares/cares/src/lib/ares__readaddrinfo.c \ - third_party/cares/cares/src/lib/ares_parse_uri_reply.c \ - third_party/cares/cares/src/lib/ares__parse_into_addrinfo.c \ + third_party/cares/cares/src/lib/ares_getsock.c \ + third_party/cares/cares/src/lib/ares_init.c \ + third_party/cares/cares/src/lib/ares_library_init.c \ + third_party/cares/cares/src/lib/ares_llist.c \ + third_party/cares/cares/src/lib/ares_mkquery.c \ + third_party/cares/cares/src/lib/ares_nowarn.c \ + third_party/cares/cares/src/lib/ares_options.c \ third_party/cares/cares/src/lib/ares_parse_a_reply.c \ third_party/cares/cares/src/lib/ares_parse_aaaa_reply.c \ - third_party/cares/cares/src/lib/ares_library_init.c \ - third_party/cares/cares/src/lib/ares_init.c \ - third_party/cares/cares/src/lib/ares_gethostbyname.c \ - third_party/cares/cares/src/lib/ares_getaddrinfo.c \ - third_party/cares/cares/src/lib/ares_freeaddrinfo.c \ - third_party/cares/cares/src/lib/ares_expand_name.c \ - third_party/cares/cares/src/lib/ares_destroy.c \ - third_party/cares/cares/src/lib/ares_data.c \ - third_party/cares/cares/src/lib/ares__addrinfo_localhost.c \ - third_party/cares/cares/src/lib/ares__addrinfo2hostent.c \ - third_party/cares/cares/src/lib/inet_net_pton.c \ - third_party/cares/cares/src/lib/ares_strsplit.c \ - third_party/cares/cares/src/lib/ares_strdup.c \ - third_party/cares/cares/src/lib/ares_send.c \ - third_party/cares/cares/src/lib/ares_rand.c \ - third_party/cares/cares/src/lib/ares_query.c \ + third_party/cares/cares/src/lib/ares_parse_caa_reply.c \ + third_party/cares/cares/src/lib/ares_parse_mx_reply.c \ + third_party/cares/cares/src/lib/ares_parse_naptr_reply.c \ + third_party/cares/cares/src/lib/ares_parse_ns_reply.c \ + third_party/cares/cares/src/lib/ares_parse_ptr_reply.c \ + third_party/cares/cares/src/lib/ares_parse_soa_reply.c \ + third_party/cares/cares/src/lib/ares_parse_srv_reply.c \ + third_party/cares/cares/src/lib/ares_parse_txt_reply.c \ + third_party/cares/cares/src/lib/ares_parse_uri_reply.c \ + third_party/cares/cares/src/lib/ares_platform.c \ third_party/cares/cares/src/lib/ares_process.c \ + third_party/cares/cares/src/lib/ares_query.c \ + third_party/cares/cares/src/lib/ares_rand.c \ + third_party/cares/cares/src/lib/ares_search.c \ + third_party/cares/cares/src/lib/ares_send.c \ + third_party/cares/cares/src/lib/ares_strcasecmp.c \ + third_party/cares/cares/src/lib/ares_strdup.c \ + third_party/cares/cares/src/lib/ares_strerror.c \ + third_party/cares/cares/src/lib/ares_strsplit.c \ + third_party/cares/cares/src/lib/ares_timeout.c \ + third_party/cares/cares/src/lib/ares_version.c \ + third_party/cares/cares/src/lib/ares_writev.c \ + third_party/cares/cares/src/lib/bitncmp.c \ + third_party/cares/cares/src/lib/inet_net_pton.c \ + third_party/cares/cares/src/lib/inet_ntop.c \ + third_party/cares/cares/src/lib/windows_port.c \ LIBCARES_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBCARES_SRC)))) @@ -3264,573 +2226,48 @@ endif # end of build recipe for library "cares" -# Add private ABSEIL target which contains all sources used by all baselib libraries. - - -# start of build recipe for library "grpc_abseil" (generated by makelib(lib) template function) +# start of build recipe for library "z" (generated by makelib(lib) template function) # deps: [] # transitive_deps: [] -LIBGRPC_ABSEIL_SRC = \ - third_party/abseil-cpp/absl/base/internal/cycleclock.cc \ - third_party/abseil-cpp/absl/base/internal/low_level_alloc.cc \ - third_party/abseil-cpp/absl/base/internal/raw_logging.cc \ - third_party/abseil-cpp/absl/base/internal/spinlock.cc \ - third_party/abseil-cpp/absl/base/internal/spinlock_wait.cc \ - third_party/abseil-cpp/absl/base/internal/strerror.cc \ - third_party/abseil-cpp/absl/base/internal/sysinfo.cc \ - third_party/abseil-cpp/absl/base/internal/thread_identity.cc \ - third_party/abseil-cpp/absl/base/internal/throw_delegate.cc \ - third_party/abseil-cpp/absl/base/internal/unscaledcycleclock.cc \ - third_party/abseil-cpp/absl/base/log_severity.cc \ - third_party/abseil-cpp/absl/container/internal/hashtablez_sampler.cc \ - third_party/abseil-cpp/absl/container/internal/hashtablez_sampler_force_weak_definition.cc \ - third_party/abseil-cpp/absl/container/internal/raw_hash_set.cc \ - third_party/abseil-cpp/absl/crc/crc32c.cc \ - third_party/abseil-cpp/absl/crc/internal/cpu_detect.cc \ - third_party/abseil-cpp/absl/crc/internal/crc.cc \ - third_party/abseil-cpp/absl/crc/internal/crc_cord_state.cc \ - third_party/abseil-cpp/absl/crc/internal/crc_memcpy_fallback.cc \ - third_party/abseil-cpp/absl/crc/internal/crc_memcpy_x86_arm_combined.cc \ - third_party/abseil-cpp/absl/crc/internal/crc_non_temporal_memcpy.cc \ - third_party/abseil-cpp/absl/crc/internal/crc_x86_arm_combined.cc \ - third_party/abseil-cpp/absl/debugging/internal/address_is_readable.cc \ - third_party/abseil-cpp/absl/debugging/internal/demangle.cc \ - third_party/abseil-cpp/absl/debugging/internal/elf_mem_image.cc \ - third_party/abseil-cpp/absl/debugging/internal/vdso_support.cc \ - third_party/abseil-cpp/absl/debugging/stacktrace.cc \ - third_party/abseil-cpp/absl/debugging/symbolize.cc \ - third_party/abseil-cpp/absl/flags/commandlineflag.cc \ - third_party/abseil-cpp/absl/flags/internal/commandlineflag.cc \ - third_party/abseil-cpp/absl/flags/internal/flag.cc \ - third_party/abseil-cpp/absl/flags/internal/private_handle_accessor.cc \ - third_party/abseil-cpp/absl/flags/internal/program_name.cc \ - third_party/abseil-cpp/absl/flags/marshalling.cc \ - third_party/abseil-cpp/absl/flags/reflection.cc \ - third_party/abseil-cpp/absl/flags/usage_config.cc \ - third_party/abseil-cpp/absl/hash/internal/city.cc \ - third_party/abseil-cpp/absl/hash/internal/hash.cc \ - third_party/abseil-cpp/absl/hash/internal/low_level_hash.cc \ - third_party/abseil-cpp/absl/numeric/int128.cc \ - third_party/abseil-cpp/absl/profiling/internal/exponential_biased.cc \ - third_party/abseil-cpp/absl/random/discrete_distribution.cc \ - third_party/abseil-cpp/absl/random/gaussian_distribution.cc \ - third_party/abseil-cpp/absl/random/internal/pool_urbg.cc \ - third_party/abseil-cpp/absl/random/internal/randen.cc \ - third_party/abseil-cpp/absl/random/internal/randen_detect.cc \ - third_party/abseil-cpp/absl/random/internal/randen_hwaes.cc \ - third_party/abseil-cpp/absl/random/internal/randen_round_keys.cc \ - third_party/abseil-cpp/absl/random/internal/randen_slow.cc \ - third_party/abseil-cpp/absl/random/internal/seed_material.cc \ - third_party/abseil-cpp/absl/random/seed_gen_exception.cc \ - third_party/abseil-cpp/absl/random/seed_sequences.cc \ - third_party/abseil-cpp/absl/status/internal/status_internal.cc \ - third_party/abseil-cpp/absl/status/status.cc \ - third_party/abseil-cpp/absl/status/status_payload_printer.cc \ - third_party/abseil-cpp/absl/status/statusor.cc \ - third_party/abseil-cpp/absl/strings/ascii.cc \ - third_party/abseil-cpp/absl/strings/charconv.cc \ - third_party/abseil-cpp/absl/strings/cord.cc \ - third_party/abseil-cpp/absl/strings/cord_analysis.cc \ - third_party/abseil-cpp/absl/strings/cord_buffer.cc \ - third_party/abseil-cpp/absl/strings/escaping.cc \ - third_party/abseil-cpp/absl/strings/internal/charconv_bigint.cc \ - third_party/abseil-cpp/absl/strings/internal/charconv_parse.cc \ - third_party/abseil-cpp/absl/strings/internal/cord_internal.cc \ - third_party/abseil-cpp/absl/strings/internal/cord_rep_btree.cc \ - third_party/abseil-cpp/absl/strings/internal/cord_rep_btree_navigator.cc \ - third_party/abseil-cpp/absl/strings/internal/cord_rep_btree_reader.cc \ - third_party/abseil-cpp/absl/strings/internal/cord_rep_consume.cc \ - third_party/abseil-cpp/absl/strings/internal/cord_rep_crc.cc \ - third_party/abseil-cpp/absl/strings/internal/cordz_functions.cc \ - third_party/abseil-cpp/absl/strings/internal/cordz_handle.cc \ - third_party/abseil-cpp/absl/strings/internal/cordz_info.cc \ - third_party/abseil-cpp/absl/strings/internal/damerau_levenshtein_distance.cc \ - third_party/abseil-cpp/absl/strings/internal/escaping.cc \ - third_party/abseil-cpp/absl/strings/internal/memutil.cc \ - third_party/abseil-cpp/absl/strings/internal/ostringstream.cc \ - third_party/abseil-cpp/absl/strings/internal/str_format/arg.cc \ - third_party/abseil-cpp/absl/strings/internal/str_format/bind.cc \ - third_party/abseil-cpp/absl/strings/internal/str_format/extension.cc \ - third_party/abseil-cpp/absl/strings/internal/str_format/float_conversion.cc \ - third_party/abseil-cpp/absl/strings/internal/str_format/output.cc \ - third_party/abseil-cpp/absl/strings/internal/str_format/parser.cc \ - third_party/abseil-cpp/absl/strings/internal/stringify_sink.cc \ - third_party/abseil-cpp/absl/strings/internal/utf8.cc \ - third_party/abseil-cpp/absl/strings/match.cc \ - third_party/abseil-cpp/absl/strings/numbers.cc \ - third_party/abseil-cpp/absl/strings/str_cat.cc \ - third_party/abseil-cpp/absl/strings/str_replace.cc \ - third_party/abseil-cpp/absl/strings/str_split.cc \ - third_party/abseil-cpp/absl/strings/string_view.cc \ - third_party/abseil-cpp/absl/strings/substitute.cc \ - third_party/abseil-cpp/absl/synchronization/barrier.cc \ - third_party/abseil-cpp/absl/synchronization/blocking_counter.cc \ - third_party/abseil-cpp/absl/synchronization/internal/create_thread_identity.cc \ - third_party/abseil-cpp/absl/synchronization/internal/futex_waiter.cc \ - third_party/abseil-cpp/absl/synchronization/internal/graphcycles.cc \ - third_party/abseil-cpp/absl/synchronization/internal/kernel_timeout.cc \ - third_party/abseil-cpp/absl/synchronization/internal/per_thread_sem.cc \ - third_party/abseil-cpp/absl/synchronization/internal/pthread_waiter.cc \ - third_party/abseil-cpp/absl/synchronization/internal/sem_waiter.cc \ - third_party/abseil-cpp/absl/synchronization/internal/stdcpp_waiter.cc \ - third_party/abseil-cpp/absl/synchronization/internal/waiter_base.cc \ - third_party/abseil-cpp/absl/synchronization/internal/win32_waiter.cc \ - third_party/abseil-cpp/absl/synchronization/mutex.cc \ - third_party/abseil-cpp/absl/synchronization/notification.cc \ - third_party/abseil-cpp/absl/time/civil_time.cc \ - third_party/abseil-cpp/absl/time/clock.cc \ - third_party/abseil-cpp/absl/time/duration.cc \ - third_party/abseil-cpp/absl/time/format.cc \ - third_party/abseil-cpp/absl/time/internal/cctz/src/civil_time_detail.cc \ - third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_fixed.cc \ - third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_format.cc \ - third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_if.cc \ - third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_impl.cc \ - third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_info.cc \ - third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_libc.cc \ - third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_lookup.cc \ - third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_posix.cc \ - third_party/abseil-cpp/absl/time/internal/cctz/src/zone_info_source.cc \ - third_party/abseil-cpp/absl/time/time.cc \ - third_party/abseil-cpp/absl/types/bad_optional_access.cc \ - third_party/abseil-cpp/absl/types/bad_variant_access.cc \ +LIBZ_SRC = \ + third_party/zlib/adler32.c \ + third_party/zlib/compress.c \ + third_party/zlib/crc32.c \ + third_party/zlib/deflate.c \ + third_party/zlib/infback.c \ + third_party/zlib/inffast.c \ + third_party/zlib/inflate.c \ + third_party/zlib/inftrees.c \ + third_party/zlib/trees.c \ + third_party/zlib/uncompr.c \ + third_party/zlib/zutil.c \ +PUBLIC_HEADERS_C += \ -LIBGRPC_ABSEIL_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBGRPC_ABSEIL_SRC)))) +LIBZ_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBZ_SRC)))) -$(LIBGRPC_ABSEIL_OBJS): CPPFLAGS += -g -Ithird_party/abseil-cpp +$(LIBZ_OBJS): CFLAGS += -fvisibility=hidden +$(LIBZ_OBJS): CPPFLAGS += -DHAVE_UNISTD_H -# static library for "grpc_abseil" -$(LIBDIR)/$(CONFIG)/libgrpc_abseil.a: $(LIBGRPC_ABSEIL_OBJS) +# static library for "z" +$(LIBDIR)/$(CONFIG)/libz.a: $(ZLIB_MERGE_OBJS) $(E) "[AR] Creating $@" $(Q) mkdir -p `dirname $@` - $(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc_abseil.a - $(Q) $(AR) $(ARFLAGS) $(LIBDIR)/$(CONFIG)/libgrpc_abseil.a $(LIBGRPC_ABSEIL_OBJS) + $(Q) rm -f $(LIBDIR)/$(CONFIG)/libz.a + $(Q) $(AR) $(ARFLAGS) $(LIBDIR)/$(CONFIG)/libz.a $(LIBZ_OBJS) ifeq ($(SYSTEM),Darwin) - $(Q) $(RANLIB) $(RANLIBFLAGS) $(LIBDIR)/$(CONFIG)/libgrpc_abseil.a + $(Q) $(RANLIB) $(RANLIBFLAGS) $(LIBDIR)/$(CONFIG)/libz.a endif -# shared library for "grpc_abseil" +# shared library for "z" ifneq ($(NO_DEPS),true) --include $(LIBGRPC_ABSEIL_OBJS:.o=.dep) -endif -# end of build recipe for library "grpc_abseil" - - - - -# TODO(jtattermusch): is there a way to get around this hack? -ifneq ($(OPENSSL_DEP),) -# This is to ensure the embedded OpenSSL is built beforehand, properly -# installing headers to their final destination on the drive. We need this -# otherwise parallel compilation will fail if a source is compiled first. -src/core/ext/filters/rbac/rbac_filter.cc: $(OPENSSL_DEP) -src/core/ext/filters/rbac/rbac_service_config_parser.cc: $(OPENSSL_DEP) -src/core/ext/filters/server_config_selector/server_config_selector_filter.cc: $(OPENSSL_DEP) -src/core/ext/filters/stateful_session/stateful_session_filter.cc: $(OPENSSL_DEP) -src/core/ext/filters/stateful_session/stateful_session_service_config_parser.cc: $(OPENSSL_DEP) -src/core/ext/gcp/metadata_query.cc: $(OPENSSL_DEP) -src/core/ext/transport/chttp2/alpn/alpn.cc: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/admin/v3/certs.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/admin/v3/clusters.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/admin/v3/config_dump.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/admin/v3/config_dump_shared.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/admin/v3/init_dump.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/admin/v3/listeners.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/admin/v3/memory.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/admin/v3/metrics.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/admin/v3/mutex_stats.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/admin/v3/server_info.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/admin/v3/tap.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/annotations/deprecation.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/annotations/resource.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/accesslog/v3/accesslog.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/bootstrap/v3/bootstrap.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/cluster/v3/circuit_breaker.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/cluster/v3/cluster.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/cluster/v3/filter.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/cluster/v3/outlier_detection.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/common/matcher/v3/matcher.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/core/v3/address.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/core/v3/backoff.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/core/v3/base.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/core/v3/config_source.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/core/v3/event_service_config.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/core/v3/extension.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/core/v3/grpc_method_list.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/core/v3/grpc_service.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/core/v3/health_check.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/core/v3/http_service.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/core/v3/http_uri.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/core/v3/protocol.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/core/v3/proxy_protocol.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/core/v3/resolver.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/core/v3/socket_option.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/core/v3/substitution_format_string.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/core/v3/udp_socket_config.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/endpoint/v3/endpoint.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/endpoint/v3/endpoint_components.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/endpoint/v3/load_report.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/listener/v3/api_listener.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/listener/v3/listener.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/listener/v3/listener_components.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/listener/v3/quic_config.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/listener/v3/udp_listener_config.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/metrics/v3/metrics_service.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/metrics/v3/stats.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/overload/v3/overload.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/rbac/v3/rbac.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/route/v3/route.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/route/v3/route_components.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/route/v3/scoped_route.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/tap/v3/common.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/trace/v3/datadog.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/trace/v3/dynamic_ot.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/trace/v3/http_tracer.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/trace/v3/lightstep.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/trace/v3/opencensus.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/trace/v3/opentelemetry.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/trace/v3/service.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/trace/v3/skywalking.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/trace/v3/trace.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/trace/v3/xray.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/config/trace/v3/zipkin.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/data/accesslog/v3/accesslog.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/extensions/clusters/aggregate/v3/cluster.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/extensions/filters/common/fault/v3/fault.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/extensions/filters/http/fault/v3/fault.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/extensions/filters/http/rbac/v3/rbac.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/extensions/filters/http/router/v3/router.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/extensions/filters/http/stateful_session/v3/stateful_session.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/extensions/http/stateful_session/cookie/v3/cookie.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/client_side_weighted_round_robin.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/common/v3/common.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/pick_first/v3/pick_first.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/ring_hash/v3/ring_hash.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/wrr_locality/v3/wrr_locality.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/cert.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/common.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/secret.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/tls.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/extensions/upstreams/http/v3/http_protocol_options.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/service/discovery/v3/ads.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/service/discovery/v3/discovery.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/service/load_stats/v3/lrs.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/service/status/v3/csds.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/type/http/v3/cookie.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/type/http/v3/path_transformation.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/type/matcher/v3/filter_state.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/type/matcher/v3/http_inputs.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/type/matcher/v3/metadata.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/type/matcher/v3/node.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/type/matcher/v3/number.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/type/matcher/v3/path.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/type/matcher/v3/regex.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/type/matcher/v3/status_code_input.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/type/matcher/v3/string.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/type/matcher/v3/struct.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/type/matcher/v3/value.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/type/metadata/v3/metadata.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/type/tracing/v3/custom_tag.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/type/v3/hash_policy.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/type/v3/http.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/type/v3/http_status.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/type/v3/percent.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/type/v3/range.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/type/v3/ratelimit_strategy.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/type/v3/ratelimit_unit.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/type/v3/semantic_version.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/envoy/type/v3/token_bucket.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/google/api/expr/v1alpha1/checked.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/google/api/expr/v1alpha1/syntax.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/google/api/httpbody.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/opencensus/proto/trace/v1/trace_config.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/src/proto/grpc/lookup/v1/rls_config.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/udpa/annotations/migrate.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/udpa/annotations/security.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/udpa/annotations/sensitive.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/udpa/annotations/status.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/udpa/annotations/versioning.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/xds/annotations/v3/migrate.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/xds/annotations/v3/security.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/xds/annotations/v3/sensitive.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/xds/annotations/v3/status.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/xds/annotations/v3/versioning.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/xds/core/v3/authority.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/xds/core/v3/cidr.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/xds/core/v3/collection_entry.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/xds/core/v3/context_params.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/xds/core/v3/extension.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/xds/core/v3/resource.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/xds/core/v3/resource_locator.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/xds/core/v3/resource_name.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/xds/type/matcher/v3/cel.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/xds/type/matcher/v3/domain.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/xds/type/matcher/v3/http_inputs.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/xds/type/matcher/v3/ip.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/xds/type/matcher/v3/matcher.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/xds/type/matcher/v3/range.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/xds/type/matcher/v3/regex.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/xds/type/matcher/v3/string.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/xds/type/v3/cel.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/xds/type/v3/range.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upb-gen/xds/type/v3/typed_struct.upb_minitable.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/admin/v3/certs.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/admin/v3/clusters.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/admin/v3/config_dump.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/admin/v3/config_dump_shared.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/admin/v3/init_dump.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/admin/v3/listeners.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/admin/v3/memory.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/admin/v3/metrics.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/admin/v3/mutex_stats.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/admin/v3/server_info.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/admin/v3/tap.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/annotations/deprecation.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/annotations/resource.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/accesslog/v3/accesslog.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/bootstrap/v3/bootstrap.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/cluster/v3/circuit_breaker.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/cluster/v3/cluster.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/cluster/v3/filter.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/cluster/v3/outlier_detection.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/common/matcher/v3/matcher.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/core/v3/address.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/core/v3/backoff.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/core/v3/base.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/core/v3/config_source.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/core/v3/event_service_config.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/core/v3/extension.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/core/v3/grpc_method_list.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/core/v3/grpc_service.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/core/v3/health_check.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/core/v3/http_service.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/core/v3/http_uri.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/core/v3/protocol.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/core/v3/proxy_protocol.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/core/v3/resolver.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/core/v3/socket_option.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/core/v3/substitution_format_string.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/core/v3/udp_socket_config.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/endpoint/v3/endpoint.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/endpoint/v3/endpoint_components.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/endpoint/v3/load_report.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/listener/v3/api_listener.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/listener/v3/listener.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/listener/v3/listener_components.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/listener/v3/quic_config.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/listener/v3/udp_listener_config.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/metrics/v3/metrics_service.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/metrics/v3/stats.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/overload/v3/overload.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/rbac/v3/rbac.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/route/v3/route.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/route/v3/route_components.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/route/v3/scoped_route.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/tap/v3/common.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/trace/v3/datadog.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/trace/v3/dynamic_ot.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/trace/v3/http_tracer.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/trace/v3/lightstep.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/trace/v3/opencensus.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/trace/v3/opentelemetry.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/trace/v3/service.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/trace/v3/skywalking.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/trace/v3/trace.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/trace/v3/xray.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/config/trace/v3/zipkin.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/data/accesslog/v3/accesslog.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/extensions/clusters/aggregate/v3/cluster.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/extensions/filters/common/fault/v3/fault.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/extensions/filters/http/fault/v3/fault.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/extensions/filters/http/rbac/v3/rbac.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/extensions/filters/http/router/v3/router.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/extensions/filters/http/stateful_session/v3/stateful_session.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/extensions/http/stateful_session/cookie/v3/cookie.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/extensions/transport_sockets/tls/v3/cert.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/extensions/transport_sockets/tls/v3/common.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/extensions/transport_sockets/tls/v3/secret.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/extensions/transport_sockets/tls/v3/tls.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/extensions/upstreams/http/v3/http_protocol_options.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/service/discovery/v3/ads.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/service/discovery/v3/discovery.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/service/load_stats/v3/lrs.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/service/status/v3/csds.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/type/http/v3/cookie.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/type/http/v3/path_transformation.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/type/matcher/v3/filter_state.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/type/matcher/v3/http_inputs.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/type/matcher/v3/metadata.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/type/matcher/v3/node.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/type/matcher/v3/number.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/type/matcher/v3/path.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/type/matcher/v3/regex.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/type/matcher/v3/status_code_input.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/type/matcher/v3/string.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/type/matcher/v3/struct.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/type/matcher/v3/value.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/type/metadata/v3/metadata.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/type/tracing/v3/custom_tag.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/type/v3/hash_policy.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/type/v3/http.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/type/v3/http_status.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/type/v3/percent.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/type/v3/range.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/type/v3/ratelimit_strategy.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/type/v3/ratelimit_unit.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/type/v3/semantic_version.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/envoy/type/v3/token_bucket.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/google/api/annotations.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/google/api/expr/v1alpha1/checked.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/google/api/expr/v1alpha1/syntax.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/google/api/http.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/google/api/httpbody.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/google/protobuf/any.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/google/protobuf/descriptor.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/google/protobuf/duration.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/google/protobuf/empty.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/google/protobuf/struct.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/google/protobuf/timestamp.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/google/protobuf/wrappers.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/google/rpc/status.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/opencensus/proto/trace/v1/trace_config.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/src/proto/grpc/lookup/v1/rls_config.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/udpa/annotations/migrate.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/udpa/annotations/security.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/udpa/annotations/sensitive.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/udpa/annotations/status.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/udpa/annotations/versioning.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/validate/validate.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/xds/annotations/v3/migrate.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/xds/annotations/v3/security.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/xds/annotations/v3/sensitive.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/xds/annotations/v3/status.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/xds/annotations/v3/versioning.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/xds/core/v3/authority.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/xds/core/v3/cidr.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/xds/core/v3/collection_entry.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/xds/core/v3/context_params.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/xds/core/v3/extension.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/xds/core/v3/resource.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/xds/core/v3/resource_locator.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/xds/core/v3/resource_name.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/xds/type/matcher/v3/cel.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/xds/type/matcher/v3/domain.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/xds/type/matcher/v3/http_inputs.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/xds/type/matcher/v3/ip.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/xds/type/matcher/v3/matcher.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/xds/type/matcher/v3/range.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/xds/type/matcher/v3/regex.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/xds/type/matcher/v3/string.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/xds/type/v3/cel.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/xds/type/v3/range.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/upbdefs-gen/xds/type/v3/typed_struct.upbdefs.c: $(OPENSSL_DEP) -src/core/ext/xds/certificate_provider_store.cc: $(OPENSSL_DEP) -src/core/ext/xds/file_watcher_certificate_provider_factory.cc: $(OPENSSL_DEP) -src/core/ext/xds/xds_api.cc: $(OPENSSL_DEP) -src/core/ext/xds/xds_audit_logger_registry.cc: $(OPENSSL_DEP) -src/core/ext/xds/xds_bootstrap.cc: $(OPENSSL_DEP) -src/core/ext/xds/xds_bootstrap_grpc.cc: $(OPENSSL_DEP) -src/core/ext/xds/xds_certificate_provider.cc: $(OPENSSL_DEP) -src/core/ext/xds/xds_channel_stack_modifier.cc: $(OPENSSL_DEP) -src/core/ext/xds/xds_client.cc: $(OPENSSL_DEP) -src/core/ext/xds/xds_client_grpc.cc: $(OPENSSL_DEP) -src/core/ext/xds/xds_client_stats.cc: $(OPENSSL_DEP) -src/core/ext/xds/xds_cluster.cc: $(OPENSSL_DEP) -src/core/ext/xds/xds_cluster_specifier_plugin.cc: $(OPENSSL_DEP) -src/core/ext/xds/xds_common_types.cc: $(OPENSSL_DEP) -src/core/ext/xds/xds_endpoint.cc: $(OPENSSL_DEP) -src/core/ext/xds/xds_health_status.cc: $(OPENSSL_DEP) -src/core/ext/xds/xds_http_fault_filter.cc: $(OPENSSL_DEP) -src/core/ext/xds/xds_http_filters.cc: $(OPENSSL_DEP) -src/core/ext/xds/xds_http_rbac_filter.cc: $(OPENSSL_DEP) -src/core/ext/xds/xds_http_stateful_session_filter.cc: $(OPENSSL_DEP) -src/core/ext/xds/xds_lb_policy_registry.cc: $(OPENSSL_DEP) -src/core/ext/xds/xds_listener.cc: $(OPENSSL_DEP) -src/core/ext/xds/xds_route_config.cc: $(OPENSSL_DEP) -src/core/ext/xds/xds_routing.cc: $(OPENSSL_DEP) -src/core/ext/xds/xds_server_config_fetcher.cc: $(OPENSSL_DEP) -src/core/ext/xds/xds_transport_grpc.cc: $(OPENSSL_DEP) -src/core/lib/gprpp/posix/directory_reader.cc: $(OPENSSL_DEP) -src/core/lib/gprpp/windows/directory_reader.cc: $(OPENSSL_DEP) -src/core/lib/http/httpcli_security_connector.cc: $(OPENSSL_DEP) -src/core/lib/json/json_util.cc: $(OPENSSL_DEP) -src/core/lib/matchers/matchers.cc: $(OPENSSL_DEP) -src/core/lib/security/authorization/audit_logging.cc: $(OPENSSL_DEP) -src/core/lib/security/authorization/grpc_authorization_engine.cc: $(OPENSSL_DEP) -src/core/lib/security/authorization/matchers.cc: $(OPENSSL_DEP) -src/core/lib/security/authorization/rbac_policy.cc: $(OPENSSL_DEP) -src/core/lib/security/authorization/stdout_logger.cc: $(OPENSSL_DEP) -src/core/lib/security/credentials/alts/alts_credentials.cc: $(OPENSSL_DEP) -src/core/lib/security/credentials/channel_creds_registry_init.cc: $(OPENSSL_DEP) -src/core/lib/security/credentials/external/aws_external_account_credentials.cc: $(OPENSSL_DEP) -src/core/lib/security/credentials/external/aws_request_signer.cc: $(OPENSSL_DEP) -src/core/lib/security/credentials/external/external_account_credentials.cc: $(OPENSSL_DEP) -src/core/lib/security/credentials/external/file_external_account_credentials.cc: $(OPENSSL_DEP) -src/core/lib/security/credentials/external/url_external_account_credentials.cc: $(OPENSSL_DEP) -src/core/lib/security/credentials/google_default/credentials_generic.cc: $(OPENSSL_DEP) -src/core/lib/security/credentials/google_default/google_default_credentials.cc: $(OPENSSL_DEP) -src/core/lib/security/credentials/iam/iam_credentials.cc: $(OPENSSL_DEP) -src/core/lib/security/credentials/jwt/json_token.cc: $(OPENSSL_DEP) -src/core/lib/security/credentials/jwt/jwt_credentials.cc: $(OPENSSL_DEP) -src/core/lib/security/credentials/jwt/jwt_verifier.cc: $(OPENSSL_DEP) -src/core/lib/security/credentials/local/local_credentials.cc: $(OPENSSL_DEP) -src/core/lib/security/credentials/oauth2/oauth2_credentials.cc: $(OPENSSL_DEP) -src/core/lib/security/credentials/ssl/ssl_credentials.cc: $(OPENSSL_DEP) -src/core/lib/security/credentials/tls/grpc_tls_certificate_distributor.cc: $(OPENSSL_DEP) -src/core/lib/security/credentials/tls/grpc_tls_certificate_match.cc: $(OPENSSL_DEP) -src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.cc: $(OPENSSL_DEP) -src/core/lib/security/credentials/tls/grpc_tls_certificate_verifier.cc: $(OPENSSL_DEP) -src/core/lib/security/credentials/tls/grpc_tls_credentials_options.cc: $(OPENSSL_DEP) -src/core/lib/security/credentials/tls/grpc_tls_crl_provider.cc: $(OPENSSL_DEP) -src/core/lib/security/credentials/tls/tls_credentials.cc: $(OPENSSL_DEP) -src/core/lib/security/credentials/xds/xds_credentials.cc: $(OPENSSL_DEP) -src/core/lib/security/security_connector/alts/alts_security_connector.cc: $(OPENSSL_DEP) -src/core/lib/security/security_connector/local/local_security_connector.cc: $(OPENSSL_DEP) -src/core/lib/security/security_connector/ssl/ssl_security_connector.cc: $(OPENSSL_DEP) -src/core/lib/security/security_connector/ssl_utils.cc: $(OPENSSL_DEP) -src/core/lib/security/security_connector/tls/tls_security_connector.cc: $(OPENSSL_DEP) -src/core/load_balancing/ring_hash/ring_hash.cc: $(OPENSSL_DEP) -src/core/load_balancing/xds/cds.cc: $(OPENSSL_DEP) -src/core/load_balancing/xds/xds_cluster_impl.cc: $(OPENSSL_DEP) -src/core/load_balancing/xds/xds_cluster_manager.cc: $(OPENSSL_DEP) -src/core/load_balancing/xds/xds_override_host.cc: $(OPENSSL_DEP) -src/core/load_balancing/xds/xds_wrr_locality.cc: $(OPENSSL_DEP) -src/core/plugin_registry/grpc_plugin_registry_extra.cc: $(OPENSSL_DEP) -src/core/resolver/google_c2p/google_c2p_resolver.cc: $(OPENSSL_DEP) -src/core/resolver/xds/xds_dependency_manager.cc: $(OPENSSL_DEP) -src/core/resolver/xds/xds_resolver.cc: $(OPENSSL_DEP) -src/core/resolver/xds/xds_resolver_trace.cc: $(OPENSSL_DEP) -src/core/tsi/alts/crypt/aes_gcm.cc: $(OPENSSL_DEP) -src/core/tsi/alts/crypt/gsec.cc: $(OPENSSL_DEP) -src/core/tsi/alts/frame_protector/alts_counter.cc: $(OPENSSL_DEP) -src/core/tsi/alts/frame_protector/alts_crypter.cc: $(OPENSSL_DEP) -src/core/tsi/alts/frame_protector/alts_frame_protector.cc: $(OPENSSL_DEP) -src/core/tsi/alts/frame_protector/alts_record_protocol_crypter_common.cc: $(OPENSSL_DEP) -src/core/tsi/alts/frame_protector/alts_seal_privacy_integrity_crypter.cc: $(OPENSSL_DEP) -src/core/tsi/alts/frame_protector/alts_unseal_privacy_integrity_crypter.cc: $(OPENSSL_DEP) -src/core/tsi/alts/frame_protector/frame_handler.cc: $(OPENSSL_DEP) -src/core/tsi/alts/handshaker/alts_handshaker_client.cc: $(OPENSSL_DEP) -src/core/tsi/alts/handshaker/alts_shared_resource.cc: $(OPENSSL_DEP) -src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc: $(OPENSSL_DEP) -src/core/tsi/alts/handshaker/alts_tsi_utils.cc: $(OPENSSL_DEP) -src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_integrity_only_record_protocol.cc: $(OPENSSL_DEP) -src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_privacy_integrity_record_protocol.cc: $(OPENSSL_DEP) -src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_record_protocol_common.cc: $(OPENSSL_DEP) -src/core/tsi/alts/zero_copy_frame_protector/alts_iovec_record_protocol.cc: $(OPENSSL_DEP) -src/core/tsi/alts/zero_copy_frame_protector/alts_zero_copy_grpc_protector.cc: $(OPENSSL_DEP) -src/core/tsi/ssl/key_logging/ssl_key_logging.cc: $(OPENSSL_DEP) -src/core/tsi/ssl/session_cache/ssl_session_boringssl.cc: $(OPENSSL_DEP) -src/core/tsi/ssl/session_cache/ssl_session_cache.cc: $(OPENSSL_DEP) -src/core/tsi/ssl/session_cache/ssl_session_openssl.cc: $(OPENSSL_DEP) -src/core/tsi/ssl_transport_security.cc: $(OPENSSL_DEP) -src/core/tsi/ssl_transport_security_utils.cc: $(OPENSSL_DEP) +-include $(LIBZ_OBJS:.o=.dep) endif +# end of build recipe for library "z" + + + .PHONY: all strip tools dep_error openssl_dep_error openssl_dep_message git_update stop buildtests buildtests_c buildtests_cxx test test_c test_cxx install install_c install_cxx install-static install-certs strip strip-shared strip-static strip_c strip-shared_c strip-static_c strip_cxx strip-shared_cxx strip-static_cxx dep_c dep_cxx bins_dep_c bins_dep_cxx clean diff --git a/Rakefile b/Rakefile index b3b67b1a39e..acd9df2db78 100755 --- a/Rakefile +++ b/Rakefile @@ -109,7 +109,6 @@ task 'dlls', [:plat] do |t, args| env += 'SYSTEM=MINGW32 ' env += 'EMBED_ZLIB=true ' env += 'EMBED_OPENSSL=true ' - env += 'EMBED_CARES=true ' env += 'BUILDDIR=/tmp ' env += "V=#{verbose} " env += "GRPC_RUBY_BUILD_PROCS=#{nproc_override} " diff --git a/build_handwritten.yaml b/build_handwritten.yaml index 6282f4993dc..82e9226e270 100644 --- a/build_handwritten.yaml +++ b/build_handwritten.yaml @@ -129,10 +129,6 @@ configs: test_environ: UBSAN_OPTIONS: halt_on_error=1:print_stacktrace=1:suppressions=test/core/util/ubsan_suppressions.txt defaults: - abseil: - CPPFLAGS: -g -Ithird_party/abseil-cpp - benchmark: - CPPFLAGS: -Ithird_party/benchmark/include -DHAVE_POSIX_REGEX boringssl: CFLAGS: -g CPPFLAGS: -Ithird_party/boringssl-with-bazel/src/include -fvisibility=hidden -DOPENSSL_NO_ASM @@ -150,7 +146,8 @@ defaults: COREFLAGS: -fno-exceptions CPPFLAGS: -g -Wall -Wextra -DOSATOMIC_USE_INLINED=1 -Ithird_party/abseil-cpp -Ithird_party/re2 -Ithird_party/upb -Isrc/core/ext/upb-gen -Isrc/core/ext/upbdefs-gen -Ithird_party/utf8_range - -Ithird_party/xxhash + -Ithird_party/xxhash -Ithird_party/cares/cares/include -Ithird_party/cares -Ithird_party/cares/cares + -Ithird_party/address_sorting/include LDFLAGS: -g zlib: CFLAGS: -fvisibility=hidden diff --git a/grpc.gyp b/grpc.gyp index bb844663ee0..233e3be63a5 100644 --- a/grpc.gyp +++ b/grpc.gyp @@ -62,6 +62,10 @@ '-Isrc/core/ext/upbdefs-gen', '-Ithird_party/utf8_range', '-Ithird_party/xxhash', + '-Ithird_party/cares/cares/include', + '-Ithird_party/cares', + '-Ithird_party/cares/cares', + '-Ithird_party/address_sorting/include', ], 'ldflags': [ '-g', @@ -143,6 +147,10 @@ '-Isrc/core/ext/upbdefs-gen', '-Ithird_party/utf8_range', '-Ithird_party/xxhash', + '-Ithird_party/cares/cares/include', + '-Ithird_party/cares', + '-Ithird_party/cares/cares', + '-Ithird_party/address_sorting/include', ], 'OTHER_CPLUSPLUSFLAGS': [ '-g', @@ -156,6 +164,10 @@ '-Isrc/core/ext/upbdefs-gen', '-Ithird_party/utf8_range', '-Ithird_party/xxhash', + '-Ithird_party/cares/cares/include', + '-Ithird_party/cares', + '-Ithird_party/cares/cares', + '-Ithird_party/address_sorting/include', '-stdlib=libc++', '-std=c++14', '-Wno-error=deprecated-declarations', diff --git a/src/php/ext/grpc/config.m4 b/src/php/ext/grpc/config.m4 index 5600df34ccf..bfef03e3b55 100755 --- a/src/php/ext/grpc/config.m4 +++ b/src/php/ext/grpc/config.m4 @@ -60,17 +60,6 @@ if test "$PHP_GRPC" != "no"; then PHP_ADD_LIBPATH($GRPC_LIBDIR) - PHP_CHECK_LIBRARY(gpr,gpr_now, - [ - PHP_ADD_LIBRARY(gpr,,GRPC_SHARED_LIBADD) - PHP_ADD_LIBRARY(gpr) - AC_DEFINE(HAVE_GPRLIB,1,[ ]) - ],[ - AC_MSG_ERROR([wrong gpr lib version or lib not found]) - ],[ - -L$GRPC_LIBDIR - ]) - PHP_CHECK_LIBRARY(grpc,grpc_channel_destroy, [ PHP_ADD_LIBRARY(grpc,,GRPC_SHARED_LIBADD) diff --git a/src/ruby/ext/grpc/extconf.rb b/src/ruby/ext/grpc/extconf.rb index 18e8a919c53..ba53b0f50a0 100644 --- a/src/ruby/ext/grpc/extconf.rb +++ b/src/ruby/ext/grpc/extconf.rb @@ -100,8 +100,6 @@ ENV['EMBED_OPENSSL'] = (RUBY_ENGINE != 'truffleruby').to_s # Don't embed on TruffleRuby (the system zlib is already linked for the zlib C extension, slow build times) ENV['EMBED_ZLIB'] = (RUBY_ENGINE != 'truffleruby').to_s -ENV['EMBED_CARES'] = 'true' - ENV['ARCH_FLAGS'] = RbConfig::CONFIG['ARCH_FLAG'] if apple_toolchain && !cross_compiling if RUBY_PLATFORM =~ /arm64/ diff --git a/templates/CMakeLists.txt.template b/templates/CMakeLists.txt.template index 08cab68e7f4..4362aa44878 100644 --- a/templates/CMakeLists.txt.template +++ b/templates/CMakeLists.txt.template @@ -197,19 +197,6 @@ deps.append(d) return deps - def get_all_deps(target_dict, all_libs): - ret = set() - get_all_deps_recurse(target_dict, all_libs, ret) - return ret - - def get_all_deps_recurse(target_dict, all_libs, all_deps): - for d in target_dict.get('deps', []): - if not d in all_deps: - all_deps.add(d) - for other_lib in all_libs: - if other_lib.name == d: - get_all_deps_recurse(other_lib, all_libs, all_deps) - def is_generate_cmake_target(lib_or_target): """Returns True if a cmake target should be generated for given library/target.""" # TODO(jtattermusch): extract the metadata to a centralized location. @@ -844,11 +831,11 @@ dll_annotations.append("GRPC_DLL_EXPORTS") if lib.name in grpcxx_libs: dll_annotations.append("GRPCXX_DLL_EXPORTS") - if set(gpr_libs) & set(get_all_deps(lib, libs)): + if set(gpr_libs) & set(lib.transitive_deps): dll_annotations.append("GPR_DLL_IMPORTS") - if set(grpc_libs) & set(get_all_deps(lib, libs)): + if set(grpc_libs) & set(lib.transitive_deps): dll_annotations.append("GRPC_DLL_IMPORTS") - if set(grpcxx_libs) & set(get_all_deps(lib, libs)): + if set(grpcxx_libs) & set(lib.transitive_deps): dll_annotations.append("GRPCXX_DLL_IMPORTS") %> % if dll_annotations: @@ -945,11 +932,11 @@ % endfor )<% dll_annotations = [] - if set(gpr_libs) & set(get_all_deps(tgt, libs)): + if set(gpr_libs) & set(tgt.transitive_deps): dll_annotations.append("GPR_DLL_IMPORTS") - if set(grpc_libs) & set(get_all_deps(tgt, libs)): + if set(grpc_libs) & set(tgt.transitive_deps): dll_annotations.append("GRPC_DLL_IMPORTS") - if set(grpcxx_libs) & set(get_all_deps(tgt, libs)): + if set(grpcxx_libs) & set(tgt.transitive_deps): dll_annotations.append("GRPCXX_DLL_IMPORTS") %> % if dll_annotations: diff --git a/templates/Makefile.template b/templates/Makefile.template index 490b3d0fb53..5095da45394 100644 --- a/templates/Makefile.template +++ b/templates/Makefile.template @@ -24,31 +24,11 @@ import re import os - def is_absl_lib(target_name): - """Returns True if the target name is one of the abseil libraries.""" - return target_name.startswith("absl/") - - - def collapse_absl_deps(deps): - """Replace first occurrence of absl dependency with grpc_abseil and remove the remaining absl dependencies.""" - result = [] - absl_dep_added = False - for dep in deps: - if is_absl_lib(dep): - if not absl_dep_added: - result.append('grpc_abseil') - absl_dep_added = True - else: - result.append(dep) - return result - def get_dep_expression(dep): """For given dependency, return the expression to be used in Makefile rule dependencies.""" if dep == 'z': return "$(ZLIB_DEP)" - elif dep == 'grpc_abseil': - return "$(GRPC_ABSEIL_DEP)" elif dep == 'libssl': return "$(OPENSSL_DEP)" else: @@ -59,8 +39,6 @@ """For given dependency, return the lib archive expression to be used when linking.""" if dep == 'z': return "$(ZLIB_MERGE_LIBS)" - elif dep == 'grpc_abseil': - return "$(GRPC_ABSEIL_MERGE_LIBS)" elif dep == 'libssl': return "$(OPENSSL_MERGE_LIBS)" else: @@ -71,8 +49,6 @@ """For given dependency, return the expression with variable that has list of all object files.""" if dep == 'z': return "$(ZLIB_MERGE_OBJS)" - elif dep == 'grpc_abseil': - return "$(LIBGRPC_ABSEIL_OBJS)" elif dep == 'libssl': return "$(OPENSSL_MERGE_OBJS)" else: @@ -83,8 +59,7 @@ """Generate make rule dependency list for given library, when building as static.""" make_rule_deps = [] - - collapsed_deps = collapse_absl_deps(lib.get('transitive_deps', [])) + collapsed_deps = lib.get('transitive_deps', []) # depend on static libraries for dep in collapsed_deps: @@ -103,7 +78,7 @@ def get_merge_objs_for_deps(lib): """Get list of merge objs for all deps of a given library.""" result = [] - collapsed_deps = collapse_absl_deps(lib.get('transitive_deps', [])) + collapsed_deps = lib.get('transitive_deps', []) for dep in collapsed_deps: result.append(get_objs_expression(dep)) @@ -115,7 +90,7 @@ """Generate make rule dependency list given library, when built as shared.""" make_rule_deps = [] - collapsed_deps = collapse_absl_deps(lib.get('transitive_deps', [])) + collapsed_deps = lib.get('transitive_deps', []) # depend of obj files of this library itself make_rule_deps.append(get_objs_expression(lib.name)) @@ -131,7 +106,7 @@ """Generate list of libraries to link given library, when built as shared.""" result = [] - collapsed_deps = collapse_absl_deps(lib.get('transitive_deps', [])) + collapsed_deps = lib.get('transitive_deps', []) # depend of obj files of this library itself result.append(get_objs_expression(lib.name)) @@ -148,37 +123,31 @@ return " ".join(result) - sources_that_need_openssl = set() - sources_that_don_t_need_openssl = set() - - # warnings we'd like, but that don't exist in all compilers - PREFERRED_WARNINGS=['extra-semi'] - CHECK_WARNINGS=PREFERRED_WARNINGS + ['no-shift-negative-value', 'no-unused-but-set-variable', 'no-maybe-uninitialized', 'no-unknown-warning-option'] - - def warning_var(fmt, warning): - return fmt % warning.replace('-', '_').replace('+', 'X').upper() - - def neg_warning(warning): - if warning[0:3] == 'no-': - return warning[3:] - else: - return 'no-' + warning - lang_to_var = { 'c': 'CORE', 'c++': 'CPP', } %> <% + lib_maps = {lib.name: lib for lib in libs} + sys_libs = ['boringssl', 'cares', 'libssl', 'z'] + + # Build a new gRPC target which embeds sources of all dependencies except system libraries such as libz + grpc_lib = lib_maps.get('grpc', None) + for dep in set(grpc_lib.transitive_deps) - set(sys_libs): + dep_lib = lib_maps.get(dep, None) + if dep_lib: + grpc_lib.src += dep_lib.src + grpc_lib.headers += dep_lib.headers + grpc_lib.src = list(sorted(set(grpc_lib.src))) + grpc_lib.headers = list(sorted(set(grpc_lib.headers))) + grpc_lib.deps = list(sorted(list(set(grpc_lib.deps) & set(sys_libs)))) + grpc_lib.transitive_deps = list(sorted(list(set(grpc_lib.transitive_deps) & set(sys_libs)))) + # Makefile is only intended for internal needs (building distribution artifacts etc.) # so we can restrict the number of libraries/targets that are buildable using the Makefile. # Other targets can be built with cmake or bazel. - # TODO(jtattermusch): Figure out how to avoid the need to list the dependencies explicitly. - # Currently it is necessary because some dependencies are marked as "build: private" in build.yaml - # (which itself is correct, as they are not "public" libraries from our perspective and cmake - # needs to have them marked as such) - filtered_libs = [lib for lib in libs if (lib.build in ['all'] and lib.language != 'c++') or lib.name in ['cares', 'boringssl', 'z']] - filtered_targets = [tgt for tgt in targets if tgt.build in ['all'] and lib.language != 'c++'] + filtered_libs = [grpc_lib, ] + [lib_maps[lib] for lib in sys_libs if lib != 'libssl'] %> comma := , @@ -317,27 +286,6 @@ $(error Invalid CONFIG value '$(CONFIG)') endif - ifeq ($(SYSTEM),Linux) - TMPOUT = /dev/null - else - TMPOUT = `mktemp /tmp/test-out-XXXXXX` - endif - - CHECK_NO_CXX14_COMPAT_WORKS_CMD = $(CC) -std=c++14 -Werror -Wno-c++14-compat -o $(TMPOUT) -c test/build/no-c++14-compat.cc - HAS_WORKING_NO_CXX14_COMPAT = $(shell $(CHECK_NO_CXX14_COMPAT_WORKS_CMD) 2> /dev/null && echo true || echo false) - ifeq ($(HAS_WORKING_NO_CXX14_COMPAT),true) - W_NO_CXX14_COMPAT=-Wno-c++14-compat - endif - - %for warning in CHECK_WARNINGS: - ${warning_var('CHECK_%s_WORKS_CMD', warning)} = $(CC) -std=c99 -Werror -W${warning} -o $(TMPOUT) -c test/build/${warning}.c - ${warning_var('HAS_WORKING_%s', warning)} = $(shell $(${warning_var('CHECK_%s_WORKS_CMD', warning)}) 2> /dev/null && echo true || echo false) - ifeq ($(${warning_var('HAS_WORKING_%s', warning)}),true) - ${warning_var('W_%s', warning)}=-W${warning} - ${warning_var('NO_W_%s', warning)}=-W${neg_warning(warning)} - endif - %endfor - # The HOST compiler settings are used to compile the protoc plugins. # In most cases, you won't have to change anything, but if you are # cross-compiling, you can override these variables from GNU make's @@ -348,7 +296,7 @@ HOST_LD ?= $(LD) HOST_LDXX ?= $(LDXX) - CFLAGS += -std=c11 ${' '.join(warning_var('$(W_%s)', warning) for warning in PREFERRED_WARNINGS)} + CFLAGS += -std=c11 CXXFLAGS += -std=c++14 ifeq ($(SYSTEM),Darwin) CXXFLAGS += -stdlib=libc++ @@ -519,36 +467,6 @@ LIBS += z endif - # Setup c-ares dependency - - ifeq ($(wildcard third_party/cares/cares/include/ares.h),) - HAS_EMBEDDED_CARES = false - else - HAS_EMBEDDED_CARES = true - endif - - ifeq ($(HAS_EMBEDDED_CARES),true) - EMBED_CARES ?= true - else - # only building with c-ares from submodule is supported - DEP_MISSING += cares - EMBED_CARES ?= broken - endif - - ifeq ($(EMBED_CARES),true) - CPPFLAGS := -Ithird_party/cares/cares/include -Ithird_party/cares -Ithird_party/cares/cares $(CPPFLAGS) - endif - - # Setup address_sorting dependency - - # TODO(jtattermusch): should the include be added elsewhere? - CPPFLAGS := -Ithird_party/address_sorting/include $(CPPFLAGS) - - # Setup abseil dependency - - GRPC_ABSEIL_DEP = $(LIBDIR)/$(CONFIG)/libgrpc_abseil.a - GRPC_ABSEIL_MERGE_LIBS = $(LIBDIR)/$(CONFIG)/libgrpc_abseil.a - # Setup boringssl dependency ifeq ($(wildcard third_party/boringssl-with-bazel/src/include/openssl/ssl.h),) @@ -576,22 +494,15 @@ LIBS_SECURE = $(OPENSSL_LIBS) endif # DISABLE_ALPN endif # EMBED_OPENSSL - + LDLIBS_SECURE += $(addprefix -l, $(LIBS_SECURE)) ifeq ($(MAKECMDGOALS),clean) NO_DEPS = true endif - .SECONDARY = %.pb.h %.pb.cc - ifeq ($(DEP_MISSING),) - all: static shared\ - % for tgt in filtered_targets: - % if tgt.build == 'all': - $(BINDIR)/$(CONFIG)/${tgt.name}\ - % endif - % endfor + all: static shared dep_error: @echo "You shouldn't see this message - all of your dependencies are correct." @@ -665,10 +576,6 @@ stop: @false - % for tgt in filtered_targets: - ${tgt.name}: $(BINDIR)/$(CONFIG)/${tgt.name} - % endfor - run_dep_checks: @echo "run_dep_checks target has been deprecated." @@ -839,11 +746,6 @@ $(Q) mkdir -p `dirname $@` $(Q) $(CXX) $(CPPFLAGS) $(CXXFLAGS) $(COREFLAGS) -MMD -MF $(addsuffix .dep, $(basename $@)) -c -o $@ $< - $(OBJDIR)/$(CONFIG)/test/core/%.o : test/core/%.cc - $(E) "[CXX] Compiling $<" - $(Q) mkdir -p `dirname $@` - $(Q) $(CXX) $(CPPFLAGS) $(CXXFLAGS) $(COREFLAGS) -MMD -MF $(addsuffix .dep, $(basename $@)) -c -o $@ $< - $(OBJDIR)/$(CONFIG)/%.o : %.cc $(E) "[CXX] Compiling $<" $(Q) mkdir -p `dirname $@` @@ -877,41 +779,13 @@ % endif % endfor - # Add private ABSEIL target which contains all sources used by all baselib libraries. - <% - # Collect all abseil source and header files used by gpr, grpc, so on. - used_abseil_rules = set() - for lib in libs: - if lib.get("baselib"): - for dep in lib.transitive_deps: - if is_absl_lib(dep): - used_abseil_rules.add(dep) - used_abseil_srcs = [] - used_abseil_hdrs = [] - for lib in libs: - if lib.name in used_abseil_rules: - used_abseil_srcs.extend(lib.get("src", [])) - used_abseil_hdrs.extend(lib.get("hdr", [])) - # Create `grpc_abseil` rule with collected files. - lib_type = type(libs[0]) - grpc_abseil_lib = lib_type({ - "name": "grpc_abseil", - "build": "private", - "language": "c", - "defaults": "abseil", - "src": sorted(used_abseil_srcs), - "hdr": sorted(used_abseil_hdrs), - }) - %> - ${makelib(grpc_abseil_lib)} - <%def name="makelib(lib)"> # start of build recipe for library "${lib.name}" (generated by makelib(lib) template function) - # deps: ${collapse_absl_deps(lib.get('deps', []))} - # transitive_deps: ${collapse_absl_deps(lib.get('transitive_deps', []))} + # deps: ${lib.get('deps', [])} + # transitive_deps: ${lib.get('transitive_deps', [])} LIB${lib.name.upper()}_SRC = \\ - % for src in lib.src: + % for src in sorted(lib.src): ${src} \\ % endfor @@ -924,7 +798,7 @@ PUBLIC_HEADERS_C += \\ % endif - % for hdr in lib.public_headers: + % for hdr in sorted(lib.public_headers): ${hdr} \\ % endfor @@ -972,13 +846,6 @@ ldflags = '$(LDFLAGS)' if lib.get('LDFLAGS', None): ldflags += ' ' + lib['LDFLAGS'] - - if 'libssl' in lib.get('transitive_deps', []): - for src in lib.src: - sources_that_need_openssl.add(src) - else: - for src in lib.src: - sources_that_don_t_need_openssl.add(src) %> # shared library for "${lib.name}" % if lib.build == "all": @@ -1019,18 +886,6 @@ # end of build recipe for library "${lib.name}" - # TODO(jtattermusch): is there a way to get around this hack? - ifneq ($(OPENSSL_DEP),) - # This is to ensure the embedded OpenSSL is built beforehand, properly - # installing headers to their final destination on the drive. We need this - # otherwise parallel compilation will fail if a source is compiled first. - % for src in sorted(sources_that_need_openssl): - % if src not in sources_that_don_t_need_openssl: - ${src}: $(OPENSSL_DEP) - % endif - % endfor - endif - .PHONY: all strip tools \ dep_error openssl_dep_error openssl_dep_message git_update stop \ buildtests buildtests_c buildtests_cxx \ diff --git a/tools/buildgen/_utils.py b/tools/buildgen/_utils.py index e6634e8a417..66373176e9f 100755 --- a/tools/buildgen/_utils.py +++ b/tools/buildgen/_utils.py @@ -34,9 +34,45 @@ def import_python_module(path: str) -> types.ModuleType: class Bunch(dict): """Allows dot-accessible dictionaries.""" - def __init__(self, d: Mapping): - dict.__init__(self, d) - self.__dict__.update(d) + def __contains__(self, k): + try: + return dict.__contains__(self, k) or hasattr(self, k) + except: + return False + + def __getattr__(self, k): + try: + # Throws exception if not in prototype chain + return object.__getattribute__(self, k) + except AttributeError: + try: + return self[k] + except KeyError: + raise AttributeError(k) + + def __setattr__(self, k, v): + try: + # Throws exception if not in prototype chain + object.__getattribute__(self, k) + except AttributeError: + try: + self[k] = v + except: + raise AttributeError(k) + else: + object.__setattr__(self, k, v) + + def __delattr__(self, k): + try: + # Throws exception if not in prototype chain + object.__getattribute__(self, k) + except AttributeError: + try: + del self[k] + except KeyError: + raise AttributeError(k) + else: + object.__delattr__(self, k) def to_bunch(var: Any) -> Any: From d1cea2dd096a18bbf00823a432aa544b025c2e11 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 7 Mar 2024 11:08:03 -0800 Subject: [PATCH 15/52] Automated fix for refs/heads/master (#36059) PanCakes to the rescue! We noticed that our 'sanity' test was going to fail, but we think we can fix that automatically, so we put together this PR to do just that! If you'd like to opt-out of these PR's, add yourself to NO_AUTOFIX_USERS in .github/workflows/pr-auto-fix.yaml Closes #36059 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/36059 from grpc:create-pull-request/patch-02276f4 fb93164443d5db29125b03b14b00bbd827b1b71f PiperOrigin-RevId: 613642099 --- test/core/client_channel/lb_policy/lb_policy_test_lib.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/core/client_channel/lb_policy/lb_policy_test_lib.h b/test/core/client_channel/lb_policy/lb_policy_test_lib.h index b7444add2b6..814492c342f 100644 --- a/test/core/client_channel/lb_policy/lb_policy_test_lib.h +++ b/test/core/client_channel/lb_policy/lb_policy_test_lib.h @@ -402,9 +402,7 @@ class LoadBalancingPolicyTest : public ::testing::Test { return test_->work_serializer_; } - ConnectivityStateTracker& state_tracker() { - return state_tracker_; - } + ConnectivityStateTracker& state_tracker() { return state_tracker_; } private: const std::string address_; From 675dcccd5e9e329a71e18936759432badb4999de Mon Sep 17 00:00:00 2001 From: Xuan Wang Date: Thu, 7 Mar 2024 15:12:35 -0800 Subject: [PATCH 16/52] [AbortError] Reapply "[AbortError] And and check AbortError while abort" (#34525) Reverts grpc/grpc#34515 This PR reapplies AbortError change as the previous one was reverted. This change was mentioned in this gRFC: [L105: Python Add New Error Types](https://github.com/grpc/proposal/blob/master/L105-python-expose-new-error-types.md) Closes #34525 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/34525 from grpc:revert-34515-revert-33969-checkAbortError ce63ab1d6a2d2fc0fff7eb2897e25ffa8c370fa2 PiperOrigin-RevId: 613718295 --- src/python/grpcio/grpc/BUILD.bazel | 6 ++ src/python/grpcio/grpc/__init__.py | 20 +++---- src/python/grpcio/grpc/_channel.py | 4 +- .../grpc/_cython/_cygrpc/aio/call.pyx.pxi | 1 + .../_cygrpc/aio/callback_common.pyx.pxi | 1 + .../grpc/_cython/_cygrpc/aio/channel.pyx.pxi | 1 + .../grpc/_cython/_cygrpc/aio/common.pyx.pxi | 24 -------- .../grpc/_cython/_cygrpc/aio/server.pyx.pxi | 2 + .../grpc/_cython/_cygrpc/server.pyx.pxi | 1 + src/python/grpcio/grpc/_errors.py | 57 +++++++++++++++++++ src/python/grpcio/grpc/_server.py | 12 +++- src/python/grpcio/grpc/aio/__init__.py | 8 +-- src/python/grpcio/grpc/aio/_call.py | 8 ++- src/python/grpcio/grpc/aio/_channel.py | 3 +- src/python/grpcio/grpc/aio/_interceptor.py | 5 +- .../grpcio_tests/tests/unit/_abort_test.py | 28 +++++++++ .../grpcio_tests/tests/unit/_api_test.py | 2 + .../client_stream_unary_interceptor_test.py | 5 +- 18 files changed, 139 insertions(+), 49 deletions(-) create mode 100644 src/python/grpcio/grpc/_errors.py diff --git a/src/python/grpcio/grpc/BUILD.bazel b/src/python/grpcio/grpc/BUILD.bazel index 75961b3effc..b1a0814dc82 100644 --- a/src/python/grpcio/grpc/BUILD.bazel +++ b/src/python/grpcio/grpc/BUILD.bazel @@ -99,6 +99,11 @@ py_library( srcs = ["_observability.py"], ) +py_library( + name = "errors", + srcs = ["_errors.py"], +) + py_library( name = "grpcio", srcs = ["__init__.py"], @@ -115,6 +120,7 @@ py_library( ":auth", ":channel", ":compression", + ":errors", ":interceptor", ":plugin_wrapping", ":server", diff --git a/src/python/grpcio/grpc/__init__.py b/src/python/grpcio/grpc/__init__.py index e0ec581f9d4..3aa88f40253 100644 --- a/src/python/grpcio/grpc/__init__.py +++ b/src/python/grpcio/grpc/__init__.py @@ -21,6 +21,9 @@ import sys from grpc import _compression from grpc._cython import cygrpc as _cygrpc +from grpc._errors import AbortError +from grpc._errors import BaseError +from grpc._errors import RpcError from grpc._runtime_protos import protos from grpc._runtime_protos import protos_and_services from grpc._runtime_protos import services @@ -307,13 +310,6 @@ class Status(abc.ABC): """ -############################# gRPC Exceptions ################################ - - -class RpcError(Exception): - """Raised by the gRPC library to indicate non-OK-status RPC termination.""" - - ############################## Shared Context ################################ @@ -1241,8 +1237,8 @@ class ServicerContext(RpcContext, metaclass=abc.ABCMeta): termination of the RPC. Raises: - Exception: An exception is always raised to signal the abortion the - RPC to the gRPC runtime. + AbortError: A grpc.AbortError is always raised to signal the abortion + the RPC to the gRPC runtime. """ raise NotImplementedError() @@ -1260,8 +1256,8 @@ class ServicerContext(RpcContext, metaclass=abc.ABCMeta): StatusCode.OK. Raises: - Exception: An exception is always raised to signal the abortion the - RPC to the gRPC runtime. + AbortError: A grpc.AbortError is always raised to signal the abortion + the RPC to the gRPC runtime. """ raise NotImplementedError() @@ -2273,6 +2269,8 @@ __all__ = ( "ServiceRpcHandler", "Server", "ServerInterceptor", + "AbortError", + "BaseError", "unary_unary_rpc_method_handler", "unary_stream_rpc_method_handler", "stream_unary_rpc_method_handler", diff --git a/src/python/grpcio/grpc/_channel.py b/src/python/grpcio/grpc/_channel.py index bf29982ca65..696c1abe98b 100644 --- a/src/python/grpcio/grpc/_channel.py +++ b/src/python/grpcio/grpc/_channel.py @@ -369,7 +369,9 @@ def _rpc_state_string(class_name: str, rpc_state: _RPCState) -> str: ) -class _InactiveRpcError(grpc.RpcError, grpc.Call, grpc.Future): +class _InactiveRpcError( + grpc.RpcError, grpc.Call, grpc.Future +): # pylint: disable=too-many-ancestors """An RPC error not tied to the execution of a particular RPC. The RPC represented by the state object must not be in-progress or diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/aio/call.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/aio/call.pyx.pxi index 00c0a29c2ab..4ae7e455145 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/aio/call.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/aio/call.pyx.pxi @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from grpc._errors import InternalError _EMPTY_FLAGS = 0 _EMPTY_MASK = 0 diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/aio/callback_common.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/aio/callback_common.pyx.pxi index 14a0098fc20..5ceabc442d6 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/aio/callback_common.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/aio/callback_common.pyx.pxi @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from grpc._errors import InternalError cdef class CallbackFailureHandler: diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/aio/channel.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/aio/channel.pyx.pxi index 4286ab1d271..95f0acd6f1f 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/aio/channel.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/aio/channel.pyx.pxi @@ -13,6 +13,7 @@ # limitations under the License. # +from grpc._errors import UsageError class _WatchConnectivityFailed(Exception): """Dedicated exception class for watch connectivity failed. diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/aio/common.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/aio/common.pyx.pxi index 0e3e8de00bf..01cd54a1d77 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/aio/common.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/aio/common.pyx.pxi @@ -82,30 +82,6 @@ _COMPRESSION_METADATA_STRING_MAPPING = { CompressionAlgorithm.gzip: 'gzip', } -class BaseError(Exception): - """The base class for exceptions generated by gRPC AsyncIO stack.""" - - -class UsageError(BaseError): - """Raised when the usage of API by applications is inappropriate. - - For example, trying to invoke RPC on a closed channel, mixing two styles - of streaming API on the client side. This exception should not be - suppressed. - """ - - -class AbortError(BaseError): - """Raised when calling abort in servicer methods. - - This exception should not be suppressed. Applications may catch it to - perform certain clean-up logic, and then re-raise it. - """ - - -class InternalError(BaseError): - """Raised upon unexpected errors in native code.""" - def schedule_coro_threadsafe(object coro, object loop): try: diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi index d166bd9fabf..05838fcb260 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi @@ -13,6 +13,8 @@ # limitations under the License. +from grpc._errors import BaseError, AbortError, InternalError, UsageError + import inspect import traceback import functools diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi index 29dabec61d9..146f90e48de 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from grpc._errors import InternalError, UsageError cdef class Server: diff --git a/src/python/grpcio/grpc/_errors.py b/src/python/grpcio/grpc/_errors.py new file mode 100644 index 00000000000..91ca2a89426 --- /dev/null +++ b/src/python/grpcio/grpc/_errors.py @@ -0,0 +1,57 @@ +# Copyright 2024 The gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +############################# gRPC Exceptions ################################ + + +class BaseError(Exception): + """ + The base class for exceptions generated by gRPC. + """ + + +class UsageError(BaseError): + """ + Raised when the usage of API by applications is inappropriate. + For example, trying to invoke RPC on a closed channel, mixing two styles + of streaming API on the client side. This exception should not be + suppressed. + """ + + +class AbortError(BaseError): + """ + Raised when calling abort in servicer methods. + This exception should not be suppressed. Applications may catch it to + perform certain clean-up logic, and then re-raise it. + """ + + +class InternalError(BaseError): + """ + Raised upon unexpected errors in native code. + """ + + +class RpcError(BaseError): + """Raised by the gRPC library to indicate non-OK-status RPC termination.""" + + +__all__ = ( + "BaseError", + "UsageError", + "AbortError", + "InternalError", + "RpcError", +) diff --git a/src/python/grpcio/grpc/_server.py b/src/python/grpcio/grpc/_server.py index 4ea1d5e4597..8a395785dfb 100644 --- a/src/python/grpcio/grpc/_server.py +++ b/src/python/grpcio/grpc/_server.py @@ -42,6 +42,7 @@ from grpc import _common # pytype: disable=pyi-error from grpc import _compression # pytype: disable=pyi-error from grpc import _interceptor # pytype: disable=pyi-error from grpc._cython import cygrpc +from grpc._errors import AbortError from grpc._typing import ArityAgnosticMethodHandler from grpc._typing import ChannelArgumentType from grpc._typing import DeserializingFunction @@ -404,7 +405,7 @@ class _Context(grpc.ServicerContext): self._state.code = code self._state.details = _common.encode(details) self._state.aborted = True - raise Exception() + raise AbortError() def abort_with_status(self, status: grpc.Status) -> None: self._state.trailing_metadata = status.trailing_metadata @@ -557,6 +558,15 @@ def _call_behavior( except Exception as exception: # pylint: disable=broad-except with state.condition: if state.aborted: + if not isinstance(exception, AbortError): + try: + details = f"Exception happened while aborting: {exception}" + except Exception: # pylint: disable=broad-except + details = ( + "Calling abort raised unprintable Exception!" + ) + traceback.print_exc() + _LOGGER.exception(details) _abort( state, rpc_event.call, diff --git a/src/python/grpcio/grpc/aio/__init__.py b/src/python/grpcio/grpc/aio/__init__.py index a4e104ad51b..13f38c8b1bc 100644 --- a/src/python/grpcio/grpc/aio/__init__.py +++ b/src/python/grpcio/grpc/aio/__init__.py @@ -20,13 +20,13 @@ created. AsyncIO doesn't provide thread safety for most of its APIs. from typing import Any, Optional, Sequence, Tuple import grpc -from grpc._cython.cygrpc import AbortError -from grpc._cython.cygrpc import BaseError from grpc._cython.cygrpc import EOF -from grpc._cython.cygrpc import InternalError -from grpc._cython.cygrpc import UsageError from grpc._cython.cygrpc import init_grpc_aio from grpc._cython.cygrpc import shutdown_grpc_aio +from grpc._errors import AbortError +from grpc._errors import BaseError +from grpc._errors import InternalError +from grpc._errors import UsageError from ._base_call import Call from ._base_call import RpcContext diff --git a/src/python/grpcio/grpc/aio/_call.py b/src/python/grpcio/grpc/aio/_call.py index 82b0d3ce522..2c4a905aa1e 100644 --- a/src/python/grpcio/grpc/aio/_call.py +++ b/src/python/grpcio/grpc/aio/_call.py @@ -24,6 +24,8 @@ from typing import Any, AsyncIterator, Generator, Generic, Optional, Tuple import grpc from grpc import _common from grpc._cython import cygrpc +from grpc._errors import InternalError +from grpc._errors import UsageError from . import _base_call from ._metadata import Metadata @@ -337,7 +339,7 @@ class _StreamResponseMixin(Call): if self._response_style is _APIStyle.UNKNOWN: self._response_style = style elif self._response_style is not style: - raise cygrpc.UsageError(_API_STYLE_ERROR) + raise UsageError(_API_STYLE_ERROR) def cancel(self) -> bool: if super().cancel(): @@ -418,7 +420,7 @@ class _StreamRequestMixin(Call): def _raise_for_different_style(self, style: _APIStyle): if self._request_style is not style: - raise cygrpc.UsageError(_API_STYLE_ERROR) + raise UsageError(_API_STYLE_ERROR) def cancel(self) -> bool: if super().cancel(): @@ -490,7 +492,7 @@ class _StreamRequestMixin(Call): ) try: await self._cython_call.send_serialized_message(serialized_request) - except cygrpc.InternalError as err: + except InternalError as err: self._cython_call.set_internal_error(str(err)) await self._raise_for_status() except asyncio.CancelledError: diff --git a/src/python/grpcio/grpc/aio/_channel.py b/src/python/grpcio/grpc/aio/_channel.py index ea4de20965a..53644469623 100644 --- a/src/python/grpcio/grpc/aio/_channel.py +++ b/src/python/grpcio/grpc/aio/_channel.py @@ -22,6 +22,7 @@ from grpc import _common from grpc import _compression from grpc import _grpcio_metadata from grpc._cython import cygrpc +from grpc._errors import InternalError from . import _base_call from . import _base_channel @@ -431,7 +432,7 @@ class Channel(_base_channel.Channel): continue else: # Unidentified Call object - raise cygrpc.InternalError( + raise InternalError( f"Unrecognized call object: {candidate}" ) diff --git a/src/python/grpcio/grpc/aio/_interceptor.py b/src/python/grpcio/grpc/aio/_interceptor.py index e7ceb00fbbf..c2040f5ac81 100644 --- a/src/python/grpcio/grpc/aio/_interceptor.py +++ b/src/python/grpcio/grpc/aio/_interceptor.py @@ -30,6 +30,7 @@ from typing import ( import grpc from grpc._cython import cygrpc +from grpc._errors import UsageError from . import _base_call from ._call import AioRpcError @@ -562,7 +563,7 @@ class _InterceptedStreamRequestMixin: # should be expected through an iterators provided # by the caller. if self._write_to_iterator_queue is None: - raise cygrpc.UsageError(_API_STYLE_ERROR) + raise UsageError(_API_STYLE_ERROR) try: call = await self._interceptors_task @@ -588,7 +589,7 @@ class _InterceptedStreamRequestMixin: # should be expected through an iterators provided # by the caller. if self._write_to_iterator_queue is None: - raise cygrpc.UsageError(_API_STYLE_ERROR) + raise UsageError(_API_STYLE_ERROR) try: call = await self._interceptors_task diff --git a/src/python/grpcio_tests/tests/unit/_abort_test.py b/src/python/grpcio_tests/tests/unit/_abort_test.py index 46f48bd1cae..6415429ae0c 100644 --- a/src/python/grpcio_tests/tests/unit/_abort_test.py +++ b/src/python/grpcio_tests/tests/unit/_abort_test.py @@ -20,11 +20,13 @@ import unittest import weakref import grpc +from grpc import AbortError from tests.unit import test_common from tests.unit.framework.common import test_constants _ABORT = "/test/abort" +_ABORT_WITH_SERVER_CODE = "/test/abortServerCode" _ABORT_WITH_STATUS = "/test/AbortWithStatus" _INVALID_CODE = "/test/InvalidCode" @@ -58,6 +60,20 @@ def abort_unary_unary(request, servicer_context): raise Exception("This line should not be executed!") +def abort_unary_unary_with_server_error(request, servicer_context): + try: + servicer_context.abort( + grpc.StatusCode.INTERNAL, + _ABORT_DETAILS, + ) + except AbortError as err: + servicer_context.abort( + grpc.StatusCode.INTERNAL, + str(type(err).__name__), + ) + raise Exception("This line should not be executed!") + + def abort_with_status_unary_unary(request, servicer_context): servicer_context.abort_with_status( _Status( @@ -80,6 +96,10 @@ class _GenericHandler(grpc.GenericRpcHandler): def service(self, handler_call_details): if handler_call_details.method == _ABORT: return grpc.unary_unary_rpc_method_handler(abort_unary_unary) + elif handler_call_details.method == _ABORT_WITH_SERVER_CODE: + return grpc.unary_unary_rpc_method_handler( + abort_unary_unary_with_server_error + ) elif handler_call_details.method == _ABORT_WITH_STATUS: return grpc.unary_unary_rpc_method_handler( abort_with_status_unary_unary @@ -116,6 +136,14 @@ class AbortTest(unittest.TestCase): self.assertEqual(rpc_error.code(), grpc.StatusCode.INTERNAL) self.assertEqual(rpc_error.details(), _ABORT_DETAILS) + def test_server_abort_code(self): + with self.assertRaises(grpc.RpcError) as exception_context: + self._channel.unary_unary(_ABORT_WITH_SERVER_CODE)(_REQUEST) + rpc_error = exception_context.exception + + self.assertEqual(rpc_error.code(), grpc.StatusCode.INTERNAL) + self.assertEqual(rpc_error.details(), str(AbortError.__name__)) + # This test ensures that abort() does not store the raised exception, which # on Python 3 (via the `__traceback__` attribute) holds a reference to # all local vars. Storing the raised exception can prevent GC and stop the diff --git a/src/python/grpcio_tests/tests/unit/_api_test.py b/src/python/grpcio_tests/tests/unit/_api_test.py index 1824abf08e3..7f32dcf545d 100644 --- a/src/python/grpcio_tests/tests/unit/_api_test.py +++ b/src/python/grpcio_tests/tests/unit/_api_test.py @@ -59,6 +59,8 @@ class AllTest(unittest.TestCase): "ServiceRpcHandler", "Server", "ServerInterceptor", + "AbortError", + "BaseError", "LocalConnectionType", "local_channel_credentials", "local_server_credentials", diff --git a/src/python/grpcio_tests/tests_aio/unit/client_stream_unary_interceptor_test.py b/src/python/grpcio_tests/tests_aio/unit/client_stream_unary_interceptor_test.py index 106be6cc349..05d728aba42 100644 --- a/src/python/grpcio_tests/tests_aio/unit/client_stream_unary_interceptor_test.py +++ b/src/python/grpcio_tests/tests_aio/unit/client_stream_unary_interceptor_test.py @@ -17,6 +17,7 @@ import logging import unittest import grpc +from grpc._errors import UsageError from grpc.experimental import aio from src.proto.grpc.testing import messages_pb2 @@ -544,10 +545,10 @@ class TestStreamUnaryClientInterceptor(AioTestBase): call = stub.StreamingInputCall(request_iterator()) - with self.assertRaises(grpc._cython.cygrpc.UsageError): + with self.assertRaises(UsageError): await call.write(request) - with self.assertRaises(grpc._cython.cygrpc.UsageError): + with self.assertRaises(UsageError): await call.done_writing() await channel.close() From 3032b5c48d5a5bc6e3f3213c90b75adfbf36d132 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Thu, 7 Mar 2024 16:10:24 -0800 Subject: [PATCH 17/52] [OTel C++] Add CMake build support (#36063) Changes - * Add CMake build support to `grpcpp_otel_plugin`. Currently, we are only supporting the `find_package CONFIG` method for depending on `opentelemetry-cpp`. * Since, `grpcpp_otel_plugin` is an extension of gRPC, it will not be built by default. To enable building of this target, a new CMake option `gRPC_BUILD_GRPCPP_OTEL_PLUGIN` is being added. * Also add `CMakeLists.txt` to the otel example. The `otel_plugin_test` can also be built through cmake but, for the CI to work, there are some additional changes. Those will be made in an upcoming PR. Closes #36063 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/36063 from yashykt:OTelCMakeSupport 3bc783823b17ed282ae9b6b7bf8a26cedaae30f9 PiperOrigin-RevId: 613734473 --- CMakeLists.txt | 109 ++++++++++++++---- bazel/grpc_build_system.bzl | 1 - build_autogenerated.yaml | 18 +++ cmake/opentelemetry-cpp.cmake | 44 +++++++ examples/cpp/otel/CMakeLists.txt | 79 +++++++++++++ examples/cpp/otel/README.md | 9 ++ grpc.gyp | 13 +++ templates/CMakeLists.txt.template | 20 +++- templates/README.md | 1 + test/cpp/ext/csm/metadata_exchange_test.cc | 2 +- test/cpp/ext/otel/otel_plugin_test.cc | 2 +- test/cpp/ext/otel/otel_test_library.cc | 2 +- test/cpp/ext/otel/otel_test_library.h | 2 +- .../extract_metadata_from_bazel_xml.py | 8 ++ 14 files changed, 279 insertions(+), 31 deletions(-) create mode 100644 cmake/opentelemetry-cpp.cmake create mode 100644 examples/cpp/otel/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index cf8697a597f..8faf34698dd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -224,6 +224,11 @@ set(gRPC_ABSL_USED_TARGETS absl_meta ) +# The OpenTelemetry plugin support "package" build only at present. +set(gRPC_OPENTELEMETRY_PROVIDER "package") +# set(gRPC_OPENTELEMETRY_PROVIDER "module" CACHE STRING "Provider of opentelemetry library") +# set_property(CACHE gRPC_OPENTELEMETRY_PROVIDER PROPERTY STRINGS "module" "package") + set(gRPC_USE_PROTO_LITE OFF CACHE BOOL "Use the protobuf-lite library") if(UNIX) @@ -348,6 +353,11 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_POSIX) set(_gRPC_ALLTARGETS_LIBRARIES ${_gRPC_ALLTARGETS_LIBRARIES} ${_gRPC_SYSTEMD_LIBRARIES}) endif() +option(gRPC_BUILD_GRPCPP_OTEL_PLUGIN "Build grpcpp_otel_plugin" OFF) +if(gRPC_BUILD_GRPCPP_OTEL_PLUGIN) + include(cmake/opentelemetry-cpp.cmake) +endif() + # Setup external proto library at third_party/envoy-api with 2 download URLs if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/third_party/envoy-api AND gRPC_DOWNLOAD_ARCHIVES) # Download the archive via HTTP, validate the checksum, and extract to third_party/envoy-api. @@ -1610,7 +1620,6 @@ if(gRPC_INSTALL) endif() - add_library(gpr src/core/lib/config/config_vars.cc src/core/lib/config/config_vars_non_generated.cc @@ -1775,7 +1784,6 @@ if(gRPC_INSTALL) endif() - add_library(grpc src/core/client_channel/backup_poller.cc src/core/client_channel/client_channel_channelz.cc @@ -2744,7 +2752,6 @@ endif() if(gRPC_BUILD_TESTS) - add_library(grpc_test_util test/core/event_engine/test_init.cc test/core/util/build.cc @@ -2811,7 +2818,6 @@ endif() endif() if(gRPC_BUILD_TESTS) - add_library(grpc_test_util_unsecure test/core/event_engine/test_init.cc test/core/util/build.cc @@ -2876,7 +2882,6 @@ endif() endif() - add_library(grpc_unsecure src/core/client_channel/backup_poller.cc src/core/client_channel/client_channel_channelz.cc @@ -3451,7 +3456,6 @@ endif() if(gRPC_BUILD_TESTS) - add_library(gtest third_party/googletest/googlemock/src/gmock-cardinalities.cc third_party/googletest/googlemock/src/gmock-internal-utils.cc @@ -3524,7 +3528,6 @@ target_link_libraries(gtest endif() - add_library(upb_base_lib third_party/upb/upb/base/status.c ) @@ -3576,7 +3579,6 @@ if(gRPC_INSTALL) endif() - add_library(upb_json_lib ${_gRPC_STATIC_WIN32} src/core/ext/upb-gen/google/protobuf/descriptor.upb_minitable.c third_party/upb/upb/json/decode.c @@ -3664,7 +3666,6 @@ if(gRPC_INSTALL) endif() - add_library(upb_mem_lib third_party/upb/upb/mem/alloc.c third_party/upb/upb/mem/arena.c @@ -3717,7 +3718,6 @@ if(gRPC_INSTALL) endif() - add_library(upb_message_lib third_party/upb/upb/hash/common.c third_party/upb/upb/message/array.c @@ -3778,7 +3778,6 @@ if(gRPC_INSTALL) endif() - add_library(upb_textformat_lib ${_gRPC_STATIC_WIN32} src/core/ext/upb-gen/google/protobuf/descriptor.upb_minitable.c third_party/upb/upb/lex/atoi.c @@ -3865,7 +3864,6 @@ if(gRPC_INSTALL) endif() - add_library(utf8_range_lib third_party/utf8_range/naive.c third_party/utf8_range/range2-neon.c @@ -3921,7 +3919,6 @@ endif() if(gRPC_BUILD_TESTS) if(gRPC_BUILD_CODEGEN) - add_library(benchmark_helpers ${_gRPC_STATIC_WIN32} ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/echo.pb.cc ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/echo.grpc.pb.cc @@ -4004,7 +4001,6 @@ endif() endif() - add_library(grpc++ src/core/ext/transport/binder/client/binder_connector.cc src/core/ext/transport/binder/client/channel_create.cc @@ -4345,7 +4341,6 @@ if(gRPC_INSTALL) endif() - add_library(grpc++_alts src/cpp/common/alts_context.cc src/cpp/common/alts_util.cc @@ -4419,7 +4414,6 @@ if(gRPC_INSTALL) endif() - add_library(grpc++_error_details src/cpp/util/error_details.cc ) @@ -4492,7 +4486,6 @@ endif() if(gRPC_BUILD_CODEGEN) - add_library(grpc++_reflection ${_gRPC_STATIC_WIN32} ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/reflection/v1/reflection.pb.cc ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/reflection/v1/reflection.grpc.pb.cc @@ -4577,7 +4570,6 @@ endif() endif() if(gRPC_BUILD_TESTS) - add_library(grpc++_test src/cpp/client/channel_test_peer.cc ) @@ -4651,7 +4643,6 @@ endforeach() endif() if(gRPC_BUILD_TESTS) - add_library(grpc++_test_config test/cpp/util/test_config_cc.cc ) @@ -4704,7 +4695,6 @@ target_link_libraries(grpc++_test_config endif() if(gRPC_BUILD_TESTS) - add_library(grpc++_test_util src/core/lib/gpr/subprocess_posix.cc src/core/lib/gpr/subprocess_windows.cc @@ -4777,7 +4767,6 @@ target_link_libraries(grpc++_test_util endif() - add_library(grpc++_unsecure src/cpp/client/channel_cc.cc src/cpp/client/client_callback.cc @@ -5086,7 +5075,6 @@ if(gRPC_INSTALL) endif() - add_library(grpc_authorization_provider src/core/ext/upb-gen/google/protobuf/any.upb_minitable.c src/core/ext/upb-gen/google/protobuf/descriptor.upb_minitable.c @@ -5524,7 +5512,6 @@ if(gRPC_INSTALL) endif() - add_library(grpc_plugin_support src/compiler/cpp_generator.cc src/compiler/csharp_generator.cc @@ -5599,7 +5586,6 @@ endif() # grpcpp_channelz doesn't build with protobuf-lite # See https://github.com/grpc/grpc/issues/19473 if(gRPC_BUILD_CODEGEN AND NOT gRPC_USE_PROTO_LITE) - add_library(grpcpp_channelz ${_gRPC_STATIC_WIN32} ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/channelz/channelz.pb.cc ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/channelz/channelz.grpc.pb.cc @@ -5678,6 +5664,81 @@ if(gRPC_INSTALL AND NOT gRPC_USE_PROTO_LITE) ) endif() +endif() +if(gRPC_BUILD_GRPCPP_OTEL_PLUGIN) + +add_library(grpcpp_otel_plugin + src/cpp/ext/otel/otel_client_filter.cc + src/cpp/ext/otel/otel_plugin.cc + src/cpp/ext/otel/otel_server_call_tracer.cc +) + +target_compile_features(grpcpp_otel_plugin PUBLIC cxx_std_14) + +set_target_properties(grpcpp_otel_plugin PROPERTIES + VERSION ${gRPC_CPP_VERSION} + SOVERSION ${gRPC_CPP_SOVERSION} +) + +if(WIN32 AND MSVC) + set_target_properties(grpcpp_otel_plugin PROPERTIES COMPILE_PDB_NAME "grpcpp_otel_plugin" + COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + if(BUILD_SHARED_LIBS) + target_compile_definitions(grpcpp_otel_plugin + PRIVATE + "GPR_DLL_IMPORTS" + "GRPC_DLL_IMPORTS" + "GRPCXX_DLL_IMPORTS" + ) + endif() + if(gRPC_INSTALL) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/grpcpp_otel_plugin.pdb + DESTINATION ${gRPC_INSTALL_LIBDIR} OPTIONAL + ) + endif() +endif() + +target_include_directories(grpcpp_otel_plugin + PUBLIC $ $ + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} + ${_gRPC_PROTO_GENS_DIR} +) +target_link_libraries(grpcpp_otel_plugin + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc++ + opentelemetry-cpp::api +) + +foreach(_hdr + include/grpcpp/ext/otel_plugin.h +) + string(REPLACE "include/" "" _path ${_hdr}) + get_filename_component(_path ${_path} PATH) + install(FILES ${_hdr} + DESTINATION "${gRPC_INSTALL_INCLUDEDIR}/${_path}" + ) +endforeach() + + +if(gRPC_INSTALL) + install(TARGETS grpcpp_otel_plugin EXPORT gRPCTargets + RUNTIME DESTINATION ${gRPC_INSTALL_BINDIR} + BUNDLE DESTINATION ${gRPC_INSTALL_BINDIR} + LIBRARY DESTINATION ${gRPC_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${gRPC_INSTALL_LIBDIR} + ) +endif() + endif() if(gRPC_BUILD_TESTS) diff --git a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl index 6e09896b4ed..bf241fb8685 100644 --- a/bazel/grpc_build_system.bzl +++ b/bazel/grpc_build_system.bzl @@ -204,7 +204,6 @@ def grpc_cc_library( testonly = testonly, linkopts = linkopts, includes = [ - "api/include", "include", "src/core/ext/upb-gen", # Once upb code-gen issue is resolved, remove this. "src/core/ext/upbdefs-gen", # Once upb code-gen issue is resolved, remove this. diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index daafa1d41bd..967ad017f7d 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -5097,6 +5097,24 @@ libs: - src/cpp/server/channelz/channelz_service_plugin.cc deps: - grpc++ +- name: grpcpp_otel_plugin + build: plugin + language: c++ + public_headers: + - include/grpcpp/ext/otel_plugin.h + headers: + - src/cpp/ext/otel/key_value_iterable.h + - src/cpp/ext/otel/otel_call_tracer.h + - src/cpp/ext/otel/otel_client_filter.h + - src/cpp/ext/otel/otel_plugin.h + - src/cpp/ext/otel/otel_server_call_tracer.h + src: + - src/cpp/ext/otel/otel_client_filter.cc + - src/cpp/ext/otel/otel_plugin.cc + - src/cpp/ext/otel/otel_server_call_tracer.cc + deps: + - grpc++ + - opentelemetry-cpp::api targets: - name: fd_conservation_posix_test build: test diff --git a/cmake/opentelemetry-cpp.cmake b/cmake/opentelemetry-cpp.cmake new file mode 100644 index 00000000000..51a02a39e57 --- /dev/null +++ b/cmake/opentelemetry-cpp.cmake @@ -0,0 +1,44 @@ +# Copyright 2024 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if(TARGET opentelemetry-cpp::api) + # If opentelemetry is included already, skip including it. +# OpenTelemetry does not work with "module" mode at present. +# elseif(gRPC_OPENTELEMETRY_PROVIDER STREQUAL "module") +# if(NOT OPENTELEMETRY_ROOT_DIR) +# set(OPENTELEMETRY_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/opentelemetry-cpp) +# endif() +# set(BUILD_TESTING OFF) +# if(NOT gRPC_BUILD_TESTS) +# set(WITH_API_ONLY ON) +# endif() +# set(WITH_ABSEIL ON) +# include_directories(${OPENTELEMETRY_ROOT_DIR} "${OPENTELEMETRY_ROOT_DIR}/api/include") +# add_subdirectory(${OPENTELEMETRY_ROOT_DIR} third_party/opentelemetry-cpp) +# if(EXISTS "${OPENTELEMETRY_ROOT_DIR}/CMakeLists.txt") + # Unclear whether we should install OpenTelemetry along with gRPC + # if(gRPC_INSTALL AND _gRPC_INSTALL_SUPPORTED_FROM_MODULE) + # set(OPENTELEMETRY_INSTALL ON) + # endif() +# else() +# message(WARNING "gRPC_OPENTELEMETRY_PROVIDER is \"module\" but OPENTELEMETRY_ROOT_DIR is wrong") +# endif() +# if(gRPC_INSTALL AND NOT _gRPC_INSTALL_SUPPORTED_FROM_MODULE) +# message(WARNING "gRPC_INSTALL will be forced to FALSE because gRPC_OPENTELEMETRY_PROVIDER is \"module\" and CMake version (${CMAKE_VERSION}) is less than 3.13.") +# set(gRPC_INSTALL FALSE) +# endif() +elseif(gRPC_OPENTELEMETRY_PROVIDER STREQUAL "package") + find_package(opentelemetry-cpp CONFIG REQUIRED) +endif() +set(_gRPC_FIND_OPENTELEMETRY "if(NOT TARGET opentelemetry-cpp::opentelemetry_api)\n find_package(opentelemetry-cpp)\nendif()") diff --git a/examples/cpp/otel/CMakeLists.txt b/examples/cpp/otel/CMakeLists.txt new file mode 100644 index 00000000000..5a3ecaa1767 --- /dev/null +++ b/examples/cpp/otel/CMakeLists.txt @@ -0,0 +1,79 @@ +# Copyright 2018 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# cmake build file for C++ gRPC OpenTelemetry example. +# Assumes absl, protobuf, prometheus-cpp, opentelemetry-cpp and gRPC (with -DgRPC_BUILD_OPENTELEMETRY_PLUGIN=ON) have been installed using cmake. +# See cmake_externalproject/CMakeLists.txt for all-in-one cmake build +# that automatically builds all the dependencies before building helloworld. + +cmake_minimum_required(VERSION 3.13) + +project(grpc_opentelemetry_example C CXX) + +include(../cmake/common.cmake) + +# Find prometheus-cpp package +find_package(prometheus-cpp CONFIG REQUIRED) + +# Find opentelemetry-cpp package +find_package(opentelemetry-cpp CONFIG REQUIRED) + +# Proto file +get_filename_component(hw_proto "../../protos/helloworld.proto" ABSOLUTE) +get_filename_component(hw_proto_path "${hw_proto}" PATH) + +# Generated sources +set(hw_proto_srcs "${CMAKE_CURRENT_BINARY_DIR}/helloworld.pb.cc") +set(hw_proto_hdrs "${CMAKE_CURRENT_BINARY_DIR}/helloworld.pb.h") +set(hw_grpc_srcs "${CMAKE_CURRENT_BINARY_DIR}/helloworld.grpc.pb.cc") +set(hw_grpc_hdrs "${CMAKE_CURRENT_BINARY_DIR}/helloworld.grpc.pb.h") +add_custom_command( + OUTPUT "${hw_proto_srcs}" "${hw_proto_hdrs}" "${hw_grpc_srcs}" "${hw_grpc_hdrs}" + COMMAND ${_PROTOBUF_PROTOC} + ARGS --grpc_out "${CMAKE_CURRENT_BINARY_DIR}" + --cpp_out "${CMAKE_CURRENT_BINARY_DIR}" + -I "${hw_proto_path}" + --plugin=protoc-gen-grpc="${_GRPC_CPP_PLUGIN_EXECUTABLE}" + "${hw_proto}" + DEPENDS "${hw_proto}") + +# Include generated *.pb.h files +include_directories("${CMAKE_CURRENT_BINARY_DIR}") + +# hw_grpc_proto +add_library(hw_grpc_proto + ${hw_grpc_srcs} + ${hw_grpc_hdrs} + ${hw_proto_srcs} + ${hw_proto_hdrs}) +target_link_libraries(hw_grpc_proto + ${_REFLECTION} + ${_GRPC_GRPCPP} + ${_PROTOBUF_LIBPROTOBUF}) + +# Targets greeter_callback_(client|server) +foreach(_target + greeter_callback_client greeter_callback_server) + add_executable(${_target} "${_target}.cc") + target_link_libraries(${_target} + hw_grpc_proto + absl::flags + absl::flags_parse + opentelemetry-cpp::metrics + opentelemetry-cpp::prometheus_exporter + ${_REFLECTION} + ${_GRPC_GRPCPP} + gRPC::grpcpp_otel_plugin + ${_PROTOBUF_LIBPROTOBUF}) +endforeach() diff --git a/examples/cpp/otel/README.md b/examples/cpp/otel/README.md index 7720fd64819..af31efb158c 100644 --- a/examples/cpp/otel/README.md +++ b/examples/cpp/otel/README.md @@ -39,6 +39,15 @@ $ curl localhost:9465/metrics > ***NOTE:*** If the prometheus endpoint configured is overridden, please update > the target in the above curl command. +## CMake Instructions + +The following libraries need to be installed before building the example with CMake - +* absl +* protobuf +* prometheus-cpp +* opentelemetry-cpp (with the options `-DWITH_ABSEIL=ON` `-DWITH_PROMETHEUS=ON`) +* grpc (with the option `-DgRPC_BUILD_GRPCPP_OTEL_PLUGIN=ON`) + You can find a complete set of instructions for building gRPC and running the Hello World app in the [C++ Quick Start][]. diff --git a/grpc.gyp b/grpc.gyp index 233e3be63a5..1f7a89d8d52 100644 --- a/grpc.gyp +++ b/grpc.gyp @@ -2363,6 +2363,19 @@ 'src/cpp/server/channelz/channelz_service_plugin.cc', ], }, + { + 'target_name': 'grpcpp_otel_plugin', + 'type': 'static_library', + 'dependencies': [ + 'grpc++', + 'opentelemetry-cpp::api', + ], + 'sources': [ + 'src/cpp/ext/otel/otel_client_filter.cc', + 'src/cpp/ext/otel/otel_plugin.cc', + 'src/cpp/ext/otel/otel_server_call_tracer.cc', + ], + }, { 'target_name': 'boringssl', 'type': 'static_library', diff --git a/templates/CMakeLists.txt.template b/templates/CMakeLists.txt.template index 4362aa44878..e94c05d06c1 100644 --- a/templates/CMakeLists.txt.template +++ b/templates/CMakeLists.txt.template @@ -200,7 +200,7 @@ def is_generate_cmake_target(lib_or_target): """Returns True if a cmake target should be generated for given library/target.""" # TODO(jtattermusch): extract the metadata to a centralized location. - if lib_or_target.build not in ["all", "protoc", "tool", "test", "private"]: + if lib_or_target.build not in ["all", "plugin", "protoc", "tool", "test", "private"]: return False if lib_or_target.boringssl: # Don't generate target for boringssl libs or tests @@ -361,6 +361,11 @@ absl_meta ) + # The OpenTelemetry plugin support "package" build only at present. + set(gRPC_OPENTELEMETRY_PROVIDER "package") + # set(gRPC_OPENTELEMETRY_PROVIDER "module" CACHE STRING "Provider of opentelemetry library") + # set_property(CACHE gRPC_OPENTELEMETRY_PROVIDER PROPERTY STRINGS "module" "package") + set(gRPC_USE_PROTO_LITE OFF CACHE BOOL "Use the protobuf-lite library") if(UNIX) @@ -486,6 +491,11 @@ set(_gRPC_ALLTARGETS_LIBRARIES <%text>${_gRPC_ALLTARGETS_LIBRARIES} <%text>${_gRPC_SYSTEMD_LIBRARIES}) endif() + option(gRPC_BUILD_GRPCPP_OTEL_PLUGIN "Build grpcpp_otel_plugin" OFF) + if(gRPC_BUILD_GRPCPP_OTEL_PLUGIN) + include(cmake/opentelemetry-cpp.cmake) + endif() + % for external_proto_library in external_proto_libraries: % if len(external_proto_library.urls) > 1: # Setup external proto library at ${external_proto_library.destination} with ${len(external_proto_library.urls)} download URLs @@ -727,12 +737,16 @@ if not is_generate_cmake_target(lib): continue cmake_libs.append(lib) %> + % for lib in cmake_libs: % if lib.build in ["test", "private"]: if(gRPC_BUILD_TESTS) ${cc_library(lib)} endif() % else: + % if lib.build in ["plugin"]: + if(gRPC_BUILD_${lib.name.upper()}) + % endif ${cc_library(lib)} % if not lib.build in ["tool"]: % if any(proto_re.match(src) for src in lib.src): @@ -743,6 +757,9 @@ endif() % endif % endif + % if lib.build in ["plugin"]: + endif() + % endif % endif % endfor @@ -782,7 +799,6 @@ if(gRPC_BUILD_CODEGEN) % endif % endif - add_library(${lib.name}${lib_type_for_lib(lib.name)} % for src in lib.src: % if not proto_re.match(src): diff --git a/templates/README.md b/templates/README.md index 9c4626f5eb5..03e5836d133 100644 --- a/templates/README.md +++ b/templates/README.md @@ -89,6 +89,7 @@ dll: "..." # see below. Currently, the "`build`" tag have these meanings: * `"all"`: library to build on `"make all"`, and install on the system. +* `"plugin"`: library to build on `"make all"`, and install, but the corresponding CMake option defaults to off. The option needs to be declared manually at present to allow depending on third-party dependencies. * `"protoc"`: a protoc plugin to build on `"make all"` and install on the system. * `"private"`: a library to only build for tests. * `"test"`: a test binary to run on `"make test"`. diff --git a/test/cpp/ext/csm/metadata_exchange_test.cc b/test/cpp/ext/csm/metadata_exchange_test.cc index a402d4d67ad..19667491da3 100644 --- a/test/cpp/ext/csm/metadata_exchange_test.cc +++ b/test/cpp/ext/csm/metadata_exchange_test.cc @@ -19,10 +19,10 @@ #include "src/cpp/ext/csm/metadata_exchange.h" #include "absl/functional/any_invocable.h" -#include "api/include/opentelemetry/metrics/provider.h" #include "gmock/gmock.h" #include "google/cloud/opentelemetry/resource_detector.h" #include "gtest/gtest.h" +#include "opentelemetry/metrics/provider.h" #include "opentelemetry/sdk/metrics/meter_provider.h" #include "opentelemetry/sdk/metrics/metric_reader.h" diff --git a/test/cpp/ext/otel/otel_plugin_test.cc b/test/cpp/ext/otel/otel_plugin_test.cc index 20aa27f64d1..3cb5cb318ef 100644 --- a/test/cpp/ext/otel/otel_plugin_test.cc +++ b/test/cpp/ext/otel/otel_plugin_test.cc @@ -19,9 +19,9 @@ #include "src/cpp/ext/otel/otel_plugin.h" #include "absl/functional/any_invocable.h" -#include "api/include/opentelemetry/metrics/provider.h" #include "gmock/gmock.h" #include "gtest/gtest.h" +#include "opentelemetry/metrics/provider.h" #include "opentelemetry/sdk/metrics/meter_provider.h" #include "opentelemetry/sdk/metrics/metric_reader.h" diff --git a/test/cpp/ext/otel/otel_test_library.cc b/test/cpp/ext/otel/otel_test_library.cc index 6017b58f3c5..ee474a9e330 100644 --- a/test/cpp/ext/otel/otel_test_library.cc +++ b/test/cpp/ext/otel/otel_test_library.cc @@ -19,9 +19,9 @@ #include "test/cpp/ext/otel/otel_test_library.h" #include "absl/functional/any_invocable.h" -#include "api/include/opentelemetry/metrics/provider.h" #include "gmock/gmock.h" #include "gtest/gtest.h" +#include "opentelemetry/metrics/provider.h" #include "opentelemetry/sdk/metrics/export/metric_producer.h" #include "opentelemetry/sdk/metrics/meter_provider.h" #include "opentelemetry/sdk/metrics/metric_reader.h" diff --git a/test/cpp/ext/otel/otel_test_library.h b/test/cpp/ext/otel/otel_test_library.h index 303babe2d99..8e800d1e30f 100644 --- a/test/cpp/ext/otel/otel_test_library.h +++ b/test/cpp/ext/otel/otel_test_library.h @@ -20,9 +20,9 @@ #define GRPC_TEST_CPP_EXT_OTEL_OTEL_TEST_LIBRARY_H #include "absl/functional/any_invocable.h" -#include "api/include/opentelemetry/metrics/provider.h" #include "gmock/gmock.h" #include "gtest/gtest.h" +#include "opentelemetry/metrics/provider.h" #include "opentelemetry/sdk/metrics/meter_provider.h" #include "opentelemetry/sdk/metrics/metric_reader.h" diff --git a/tools/buildgen/extract_metadata_from_bazel_xml.py b/tools/buildgen/extract_metadata_from_bazel_xml.py index 23c41909ff4..85b168a9822 100755 --- a/tools/buildgen/extract_metadata_from_bazel_xml.py +++ b/tools/buildgen/extract_metadata_from_bazel_xml.py @@ -351,6 +351,10 @@ def _external_dep_name_from_bazel_dependency(bazel_dep: str) -> Optional[str]: return "protobuf" elif bazel_dep == "@com_google_protobuf//:protoc_lib": return "protoc" + elif bazel_dep == "@io_opentelemetry_cpp//api:api": + return "opentelemetry-cpp::api" + elif bazel_dep == "@io_opentelemetry_cpp//sdk/src/metrics:metrics": + return "opentelemetry-cpp::metrics" else: # Two options here: # * either this is not external dependency at all (which is fine, we will treat it as internal library) @@ -1156,6 +1160,10 @@ _BUILD_EXTRA_METADATA = { "generate_plugin_registry": True, }, "grpcpp_channelz": {"language": "c++", "build": "all"}, + "grpcpp_otel_plugin": { + "language": "c++", + "build": "plugin", + }, "grpc++_test": { "language": "c++", "build": "private", From ad7fbc1e922dfed74a857fbdcd0c30adff90a868 Mon Sep 17 00:00:00 2001 From: AJ Heller Date: Thu, 7 Mar 2024 16:21:50 -0800 Subject: [PATCH 18/52] [EventEngine] Document RunAfter can return an invalid handle for immediate execution (#36072) Closes #36072 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/36072 from drfloob:docs/runafter-immediate-invalid-handle 4ce2dd7dd1f0454335a8e86faca1f2dd7e898b14 PiperOrigin-RevId: 613737590 --- include/grpc/event_engine/event_engine.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/grpc/event_engine/event_engine.h b/include/grpc/event_engine/event_engine.h index 727fb6ddfee..93c94664bc5 100644 --- a/include/grpc/event_engine/event_engine.h +++ b/include/grpc/event_engine/event_engine.h @@ -444,6 +444,9 @@ class EventEngine : public std::enable_shared_from_this, /// /// Implementations must not execute the closure in the calling thread before /// \a RunAfter returns. + /// + /// Implementations may return a \a kInvalid handle if the callback can be + /// immediately executed, and is therefore not cancellable. virtual TaskHandle RunAfter(Duration when, Closure* closure) = 0; /// Synonymous with scheduling an alarm to run after duration \a when. /// From b15c4304f7b04e308123c01c89a084c58521271b Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Thu, 7 Mar 2024 16:27:09 -0800 Subject: [PATCH 19/52] [PSM Interop] Enable affinity_session_drain_test for cpp (#36064) Add `gamma.affinity_session_drain_test` to `grpc/core/master/linux/psm-csm` test suite. Closes #36064 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/36064 from sergiitk:psm-interop-ssa-draining a1618555ac1190749c3e2193c7bee722a83fced6 PiperOrigin-RevId: 613738983 --- tools/internal_ci/linux/psm-csm.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/internal_ci/linux/psm-csm.sh b/tools/internal_ci/linux/psm-csm.sh index bc8b6a19bdc..d2cc11a2498 100755 --- a/tools/internal_ci/linux/psm-csm.sh +++ b/tools/internal_ci/linux/psm-csm.sh @@ -161,6 +161,7 @@ main() { test_suites=( "gamma.gamma_baseline_test" "gamma.affinity_test" + "gamma.affinity_session_drain_test" "gamma.csm_observability_test" "app_net_ssa_test" ) From 9ba8bb5dd2a7ef7ae9483154bbc10288a255d390 Mon Sep 17 00:00:00 2001 From: Xuan Wang Date: Thu, 7 Mar 2024 16:30:06 -0800 Subject: [PATCH 20/52] Revert "[AbortError] And and check AbortError while abort" (#36076) This reverts commit 675dcccd5e9e329a71e18936759432badb4999de. Closes #36076 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/36076 from XuanWang-Amos:roll_back_AbortError 1db7c399036360b43ed435802b16669fa8cbccc4 PiperOrigin-RevId: 613739975 --- src/python/grpcio/grpc/BUILD.bazel | 6 -- src/python/grpcio/grpc/__init__.py | 20 ++++--- src/python/grpcio/grpc/_channel.py | 4 +- .../grpc/_cython/_cygrpc/aio/call.pyx.pxi | 1 - .../_cygrpc/aio/callback_common.pyx.pxi | 1 - .../grpc/_cython/_cygrpc/aio/channel.pyx.pxi | 1 - .../grpc/_cython/_cygrpc/aio/common.pyx.pxi | 24 ++++++++ .../grpc/_cython/_cygrpc/aio/server.pyx.pxi | 2 - .../grpc/_cython/_cygrpc/server.pyx.pxi | 1 - src/python/grpcio/grpc/_errors.py | 57 ------------------- src/python/grpcio/grpc/_server.py | 12 +--- src/python/grpcio/grpc/aio/__init__.py | 8 +-- src/python/grpcio/grpc/aio/_call.py | 8 +-- src/python/grpcio/grpc/aio/_channel.py | 3 +- src/python/grpcio/grpc/aio/_interceptor.py | 5 +- .../grpcio_tests/tests/unit/_abort_test.py | 28 --------- .../grpcio_tests/tests/unit/_api_test.py | 2 - .../client_stream_unary_interceptor_test.py | 5 +- 18 files changed, 49 insertions(+), 139 deletions(-) delete mode 100644 src/python/grpcio/grpc/_errors.py diff --git a/src/python/grpcio/grpc/BUILD.bazel b/src/python/grpcio/grpc/BUILD.bazel index b1a0814dc82..75961b3effc 100644 --- a/src/python/grpcio/grpc/BUILD.bazel +++ b/src/python/grpcio/grpc/BUILD.bazel @@ -99,11 +99,6 @@ py_library( srcs = ["_observability.py"], ) -py_library( - name = "errors", - srcs = ["_errors.py"], -) - py_library( name = "grpcio", srcs = ["__init__.py"], @@ -120,7 +115,6 @@ py_library( ":auth", ":channel", ":compression", - ":errors", ":interceptor", ":plugin_wrapping", ":server", diff --git a/src/python/grpcio/grpc/__init__.py b/src/python/grpcio/grpc/__init__.py index 3aa88f40253..e0ec581f9d4 100644 --- a/src/python/grpcio/grpc/__init__.py +++ b/src/python/grpcio/grpc/__init__.py @@ -21,9 +21,6 @@ import sys from grpc import _compression from grpc._cython import cygrpc as _cygrpc -from grpc._errors import AbortError -from grpc._errors import BaseError -from grpc._errors import RpcError from grpc._runtime_protos import protos from grpc._runtime_protos import protos_and_services from grpc._runtime_protos import services @@ -310,6 +307,13 @@ class Status(abc.ABC): """ +############################# gRPC Exceptions ################################ + + +class RpcError(Exception): + """Raised by the gRPC library to indicate non-OK-status RPC termination.""" + + ############################## Shared Context ################################ @@ -1237,8 +1241,8 @@ class ServicerContext(RpcContext, metaclass=abc.ABCMeta): termination of the RPC. Raises: - AbortError: A grpc.AbortError is always raised to signal the abortion - the RPC to the gRPC runtime. + Exception: An exception is always raised to signal the abortion the + RPC to the gRPC runtime. """ raise NotImplementedError() @@ -1256,8 +1260,8 @@ class ServicerContext(RpcContext, metaclass=abc.ABCMeta): StatusCode.OK. Raises: - AbortError: A grpc.AbortError is always raised to signal the abortion - the RPC to the gRPC runtime. + Exception: An exception is always raised to signal the abortion the + RPC to the gRPC runtime. """ raise NotImplementedError() @@ -2269,8 +2273,6 @@ __all__ = ( "ServiceRpcHandler", "Server", "ServerInterceptor", - "AbortError", - "BaseError", "unary_unary_rpc_method_handler", "unary_stream_rpc_method_handler", "stream_unary_rpc_method_handler", diff --git a/src/python/grpcio/grpc/_channel.py b/src/python/grpcio/grpc/_channel.py index 696c1abe98b..bf29982ca65 100644 --- a/src/python/grpcio/grpc/_channel.py +++ b/src/python/grpcio/grpc/_channel.py @@ -369,9 +369,7 @@ def _rpc_state_string(class_name: str, rpc_state: _RPCState) -> str: ) -class _InactiveRpcError( - grpc.RpcError, grpc.Call, grpc.Future -): # pylint: disable=too-many-ancestors +class _InactiveRpcError(grpc.RpcError, grpc.Call, grpc.Future): """An RPC error not tied to the execution of a particular RPC. The RPC represented by the state object must not be in-progress or diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/aio/call.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/aio/call.pyx.pxi index 4ae7e455145..00c0a29c2ab 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/aio/call.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/aio/call.pyx.pxi @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from grpc._errors import InternalError _EMPTY_FLAGS = 0 _EMPTY_MASK = 0 diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/aio/callback_common.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/aio/callback_common.pyx.pxi index 5ceabc442d6..14a0098fc20 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/aio/callback_common.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/aio/callback_common.pyx.pxi @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from grpc._errors import InternalError cdef class CallbackFailureHandler: diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/aio/channel.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/aio/channel.pyx.pxi index 95f0acd6f1f..4286ab1d271 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/aio/channel.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/aio/channel.pyx.pxi @@ -13,7 +13,6 @@ # limitations under the License. # -from grpc._errors import UsageError class _WatchConnectivityFailed(Exception): """Dedicated exception class for watch connectivity failed. diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/aio/common.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/aio/common.pyx.pxi index 01cd54a1d77..0e3e8de00bf 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/aio/common.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/aio/common.pyx.pxi @@ -82,6 +82,30 @@ _COMPRESSION_METADATA_STRING_MAPPING = { CompressionAlgorithm.gzip: 'gzip', } +class BaseError(Exception): + """The base class for exceptions generated by gRPC AsyncIO stack.""" + + +class UsageError(BaseError): + """Raised when the usage of API by applications is inappropriate. + + For example, trying to invoke RPC on a closed channel, mixing two styles + of streaming API on the client side. This exception should not be + suppressed. + """ + + +class AbortError(BaseError): + """Raised when calling abort in servicer methods. + + This exception should not be suppressed. Applications may catch it to + perform certain clean-up logic, and then re-raise it. + """ + + +class InternalError(BaseError): + """Raised upon unexpected errors in native code.""" + def schedule_coro_threadsafe(object coro, object loop): try: diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi index 05838fcb260..d166bd9fabf 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/aio/server.pyx.pxi @@ -13,8 +13,6 @@ # limitations under the License. -from grpc._errors import BaseError, AbortError, InternalError, UsageError - import inspect import traceback import functools diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi index 146f90e48de..29dabec61d9 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from grpc._errors import InternalError, UsageError cdef class Server: diff --git a/src/python/grpcio/grpc/_errors.py b/src/python/grpcio/grpc/_errors.py deleted file mode 100644 index 91ca2a89426..00000000000 --- a/src/python/grpcio/grpc/_errors.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2024 The gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -############################# gRPC Exceptions ################################ - - -class BaseError(Exception): - """ - The base class for exceptions generated by gRPC. - """ - - -class UsageError(BaseError): - """ - Raised when the usage of API by applications is inappropriate. - For example, trying to invoke RPC on a closed channel, mixing two styles - of streaming API on the client side. This exception should not be - suppressed. - """ - - -class AbortError(BaseError): - """ - Raised when calling abort in servicer methods. - This exception should not be suppressed. Applications may catch it to - perform certain clean-up logic, and then re-raise it. - """ - - -class InternalError(BaseError): - """ - Raised upon unexpected errors in native code. - """ - - -class RpcError(BaseError): - """Raised by the gRPC library to indicate non-OK-status RPC termination.""" - - -__all__ = ( - "BaseError", - "UsageError", - "AbortError", - "InternalError", - "RpcError", -) diff --git a/src/python/grpcio/grpc/_server.py b/src/python/grpcio/grpc/_server.py index 8a395785dfb..4ea1d5e4597 100644 --- a/src/python/grpcio/grpc/_server.py +++ b/src/python/grpcio/grpc/_server.py @@ -42,7 +42,6 @@ from grpc import _common # pytype: disable=pyi-error from grpc import _compression # pytype: disable=pyi-error from grpc import _interceptor # pytype: disable=pyi-error from grpc._cython import cygrpc -from grpc._errors import AbortError from grpc._typing import ArityAgnosticMethodHandler from grpc._typing import ChannelArgumentType from grpc._typing import DeserializingFunction @@ -405,7 +404,7 @@ class _Context(grpc.ServicerContext): self._state.code = code self._state.details = _common.encode(details) self._state.aborted = True - raise AbortError() + raise Exception() def abort_with_status(self, status: grpc.Status) -> None: self._state.trailing_metadata = status.trailing_metadata @@ -558,15 +557,6 @@ def _call_behavior( except Exception as exception: # pylint: disable=broad-except with state.condition: if state.aborted: - if not isinstance(exception, AbortError): - try: - details = f"Exception happened while aborting: {exception}" - except Exception: # pylint: disable=broad-except - details = ( - "Calling abort raised unprintable Exception!" - ) - traceback.print_exc() - _LOGGER.exception(details) _abort( state, rpc_event.call, diff --git a/src/python/grpcio/grpc/aio/__init__.py b/src/python/grpcio/grpc/aio/__init__.py index 13f38c8b1bc..a4e104ad51b 100644 --- a/src/python/grpcio/grpc/aio/__init__.py +++ b/src/python/grpcio/grpc/aio/__init__.py @@ -20,13 +20,13 @@ created. AsyncIO doesn't provide thread safety for most of its APIs. from typing import Any, Optional, Sequence, Tuple import grpc +from grpc._cython.cygrpc import AbortError +from grpc._cython.cygrpc import BaseError from grpc._cython.cygrpc import EOF +from grpc._cython.cygrpc import InternalError +from grpc._cython.cygrpc import UsageError from grpc._cython.cygrpc import init_grpc_aio from grpc._cython.cygrpc import shutdown_grpc_aio -from grpc._errors import AbortError -from grpc._errors import BaseError -from grpc._errors import InternalError -from grpc._errors import UsageError from ._base_call import Call from ._base_call import RpcContext diff --git a/src/python/grpcio/grpc/aio/_call.py b/src/python/grpcio/grpc/aio/_call.py index 2c4a905aa1e..82b0d3ce522 100644 --- a/src/python/grpcio/grpc/aio/_call.py +++ b/src/python/grpcio/grpc/aio/_call.py @@ -24,8 +24,6 @@ from typing import Any, AsyncIterator, Generator, Generic, Optional, Tuple import grpc from grpc import _common from grpc._cython import cygrpc -from grpc._errors import InternalError -from grpc._errors import UsageError from . import _base_call from ._metadata import Metadata @@ -339,7 +337,7 @@ class _StreamResponseMixin(Call): if self._response_style is _APIStyle.UNKNOWN: self._response_style = style elif self._response_style is not style: - raise UsageError(_API_STYLE_ERROR) + raise cygrpc.UsageError(_API_STYLE_ERROR) def cancel(self) -> bool: if super().cancel(): @@ -420,7 +418,7 @@ class _StreamRequestMixin(Call): def _raise_for_different_style(self, style: _APIStyle): if self._request_style is not style: - raise UsageError(_API_STYLE_ERROR) + raise cygrpc.UsageError(_API_STYLE_ERROR) def cancel(self) -> bool: if super().cancel(): @@ -492,7 +490,7 @@ class _StreamRequestMixin(Call): ) try: await self._cython_call.send_serialized_message(serialized_request) - except InternalError as err: + except cygrpc.InternalError as err: self._cython_call.set_internal_error(str(err)) await self._raise_for_status() except asyncio.CancelledError: diff --git a/src/python/grpcio/grpc/aio/_channel.py b/src/python/grpcio/grpc/aio/_channel.py index 53644469623..ea4de20965a 100644 --- a/src/python/grpcio/grpc/aio/_channel.py +++ b/src/python/grpcio/grpc/aio/_channel.py @@ -22,7 +22,6 @@ from grpc import _common from grpc import _compression from grpc import _grpcio_metadata from grpc._cython import cygrpc -from grpc._errors import InternalError from . import _base_call from . import _base_channel @@ -432,7 +431,7 @@ class Channel(_base_channel.Channel): continue else: # Unidentified Call object - raise InternalError( + raise cygrpc.InternalError( f"Unrecognized call object: {candidate}" ) diff --git a/src/python/grpcio/grpc/aio/_interceptor.py b/src/python/grpcio/grpc/aio/_interceptor.py index c2040f5ac81..e7ceb00fbbf 100644 --- a/src/python/grpcio/grpc/aio/_interceptor.py +++ b/src/python/grpcio/grpc/aio/_interceptor.py @@ -30,7 +30,6 @@ from typing import ( import grpc from grpc._cython import cygrpc -from grpc._errors import UsageError from . import _base_call from ._call import AioRpcError @@ -563,7 +562,7 @@ class _InterceptedStreamRequestMixin: # should be expected through an iterators provided # by the caller. if self._write_to_iterator_queue is None: - raise UsageError(_API_STYLE_ERROR) + raise cygrpc.UsageError(_API_STYLE_ERROR) try: call = await self._interceptors_task @@ -589,7 +588,7 @@ class _InterceptedStreamRequestMixin: # should be expected through an iterators provided # by the caller. if self._write_to_iterator_queue is None: - raise UsageError(_API_STYLE_ERROR) + raise cygrpc.UsageError(_API_STYLE_ERROR) try: call = await self._interceptors_task diff --git a/src/python/grpcio_tests/tests/unit/_abort_test.py b/src/python/grpcio_tests/tests/unit/_abort_test.py index 6415429ae0c..46f48bd1cae 100644 --- a/src/python/grpcio_tests/tests/unit/_abort_test.py +++ b/src/python/grpcio_tests/tests/unit/_abort_test.py @@ -20,13 +20,11 @@ import unittest import weakref import grpc -from grpc import AbortError from tests.unit import test_common from tests.unit.framework.common import test_constants _ABORT = "/test/abort" -_ABORT_WITH_SERVER_CODE = "/test/abortServerCode" _ABORT_WITH_STATUS = "/test/AbortWithStatus" _INVALID_CODE = "/test/InvalidCode" @@ -60,20 +58,6 @@ def abort_unary_unary(request, servicer_context): raise Exception("This line should not be executed!") -def abort_unary_unary_with_server_error(request, servicer_context): - try: - servicer_context.abort( - grpc.StatusCode.INTERNAL, - _ABORT_DETAILS, - ) - except AbortError as err: - servicer_context.abort( - grpc.StatusCode.INTERNAL, - str(type(err).__name__), - ) - raise Exception("This line should not be executed!") - - def abort_with_status_unary_unary(request, servicer_context): servicer_context.abort_with_status( _Status( @@ -96,10 +80,6 @@ class _GenericHandler(grpc.GenericRpcHandler): def service(self, handler_call_details): if handler_call_details.method == _ABORT: return grpc.unary_unary_rpc_method_handler(abort_unary_unary) - elif handler_call_details.method == _ABORT_WITH_SERVER_CODE: - return grpc.unary_unary_rpc_method_handler( - abort_unary_unary_with_server_error - ) elif handler_call_details.method == _ABORT_WITH_STATUS: return grpc.unary_unary_rpc_method_handler( abort_with_status_unary_unary @@ -136,14 +116,6 @@ class AbortTest(unittest.TestCase): self.assertEqual(rpc_error.code(), grpc.StatusCode.INTERNAL) self.assertEqual(rpc_error.details(), _ABORT_DETAILS) - def test_server_abort_code(self): - with self.assertRaises(grpc.RpcError) as exception_context: - self._channel.unary_unary(_ABORT_WITH_SERVER_CODE)(_REQUEST) - rpc_error = exception_context.exception - - self.assertEqual(rpc_error.code(), grpc.StatusCode.INTERNAL) - self.assertEqual(rpc_error.details(), str(AbortError.__name__)) - # This test ensures that abort() does not store the raised exception, which # on Python 3 (via the `__traceback__` attribute) holds a reference to # all local vars. Storing the raised exception can prevent GC and stop the diff --git a/src/python/grpcio_tests/tests/unit/_api_test.py b/src/python/grpcio_tests/tests/unit/_api_test.py index 7f32dcf545d..1824abf08e3 100644 --- a/src/python/grpcio_tests/tests/unit/_api_test.py +++ b/src/python/grpcio_tests/tests/unit/_api_test.py @@ -59,8 +59,6 @@ class AllTest(unittest.TestCase): "ServiceRpcHandler", "Server", "ServerInterceptor", - "AbortError", - "BaseError", "LocalConnectionType", "local_channel_credentials", "local_server_credentials", diff --git a/src/python/grpcio_tests/tests_aio/unit/client_stream_unary_interceptor_test.py b/src/python/grpcio_tests/tests_aio/unit/client_stream_unary_interceptor_test.py index 05d728aba42..106be6cc349 100644 --- a/src/python/grpcio_tests/tests_aio/unit/client_stream_unary_interceptor_test.py +++ b/src/python/grpcio_tests/tests_aio/unit/client_stream_unary_interceptor_test.py @@ -17,7 +17,6 @@ import logging import unittest import grpc -from grpc._errors import UsageError from grpc.experimental import aio from src.proto.grpc.testing import messages_pb2 @@ -545,10 +544,10 @@ class TestStreamUnaryClientInterceptor(AioTestBase): call = stub.StreamingInputCall(request_iterator()) - with self.assertRaises(UsageError): + with self.assertRaises(grpc._cython.cygrpc.UsageError): await call.write(request) - with self.assertRaises(UsageError): + with self.assertRaises(grpc._cython.cygrpc.UsageError): await call.done_writing() await channel.close() From bf09088b5f3a7c769d70fd0a92e74c5038cb1d21 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Fri, 8 Mar 2024 10:29:19 -0800 Subject: [PATCH 21/52] [Build] Making grpcio_tools ready for upcoming Protobuf change. (#36074) Protobuf is beginning to rely more on upb, pulling in more C files. Hence, grpcio_tool needs the following changes to absorb this. - Changing the setup.py to support both C and C++ by removing explicit language=c++. Rather it can choose the right compiler by its extension. - Adding build_extensions injection to deal with C/C++ option conflict as grpcio does. - Adding protobuf and upb directories to the build script so that it can find newly added source files in protobuf. Tested by https://github.com/grpc/grpc/pull/35796 Closes #36074 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/36074 from veblush:grpcio-tool-ready 84abce9083ffa554b3aa042edb7a4b8e1b400539 PiperOrigin-RevId: 613975137 --- .../grpc_tools/_protoc_compiler.pyx | 1 + .../python/grpcio_tools/protoc_lib_deps.py | 2 ++ tools/distrib/python/grpcio_tools/setup.py | 35 ++++++++++++++++++- tools/distrib/python/make_grpcio_tools.py | 9 ++++- 4 files changed, 45 insertions(+), 2 deletions(-) diff --git a/tools/distrib/python/grpcio_tools/grpc_tools/_protoc_compiler.pyx b/tools/distrib/python/grpcio_tools/grpc_tools/_protoc_compiler.pyx index ff68565d069..7b307e0475c 100644 --- a/tools/distrib/python/grpcio_tools/grpc_tools/_protoc_compiler.pyx +++ b/tools/distrib/python/grpcio_tools/grpc_tools/_protoc_compiler.pyx @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# distutils: language=c++ from cython.operator cimport dereference from libc cimport stdlib diff --git a/tools/distrib/python/grpcio_tools/protoc_lib_deps.py b/tools/distrib/python/grpcio_tools/protoc_lib_deps.py index 0ce3e73b662..f2dc748eac2 100644 --- a/tools/distrib/python/grpcio_tools/protoc_lib_deps.py +++ b/tools/distrib/python/grpcio_tools/protoc_lib_deps.py @@ -319,7 +319,9 @@ PROTO_FILES=[ CC_INCLUDES=[ 'third_party/abseil-cpp', + 'third_party/protobuf', 'third_party/protobuf/src', + 'third_party/protobuf/upb', 'third_party/protobuf/third_party/utf8_range' ] PROTO_INCLUDE='third_party/protobuf/src' diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py index ee026efd00c..011d43ce6f0 100644 --- a/tools/distrib/python/grpcio_tools/setup.py +++ b/tools/distrib/python/grpcio_tools/setup.py @@ -126,6 +126,40 @@ class BuildExt(build_ext.build_ext): filename = filename[: -len(orig_ext_suffix)] + new_ext_suffix return filename + def build_extensions(self): + # This special conditioning is here due to difference of compiler + # behavior in gcc and clang. The clang doesn't take --stdc++11 + # flags but gcc does. Since the setuptools of Python only support + # all C or all C++ compilation, the mix of C and C++ will crash. + # *By default*, macOS and FreBSD use clang and Linux use gcc + # + # If we are not using a permissive compiler that's OK with being + # passed wrong std flags, swap out compile function by adding a filter + # for it. + old_compile = self.compiler._compile + + def new_compile(obj, src, ext, cc_args, extra_postargs, pp_opts): + if src.endswith(".c"): + extra_postargs = [ + arg for arg in extra_postargs if not "-std=c++" in arg + ] + elif src.endswith(".cc") or src.endswith(".cpp"): + extra_postargs = [ + arg for arg in extra_postargs if not "-std=c11" in arg + ] + return old_compile(obj, src, ext, cc_args, extra_postargs, pp_opts) + + self.compiler._compile = new_compile + + try: + build_ext.build_ext.build_extensions(self) + except Exception as error: + formatted_exception = traceback.format_exc() + support.diagnose_build_ext_error(self, error, formatted_exception) + raise CommandError( + "Failed `build_ext` step:\n{}".format(formatted_exception) + ) + # There are some situations (like on Windows) where CC, CFLAGS, and LDFLAGS are # entirely ignored/dropped/forgotten by distutils and its Cygwin/MinGW support. @@ -259,7 +293,6 @@ def extension_modules(): os.path.join("grpc_root", "include"), ] + CC_INCLUDES, - language="c++", define_macros=list(DEFINE_MACROS), extra_compile_args=list(EXTRA_COMPILE_ARGS), extra_link_args=list(EXTRA_LINK_ARGS), diff --git a/tools/distrib/python/make_grpcio_tools.py b/tools/distrib/python/make_grpcio_tools.py index 58f0ff31038..efd8b447e12 100755 --- a/tools/distrib/python/make_grpcio_tools.py +++ b/tools/distrib/python/make_grpcio_tools.py @@ -71,7 +71,9 @@ PROTOBUF_PROTO_PREFIX = "@com_google_protobuf//src/" # will be added to include path when building grpcio_tools CC_INCLUDES = [ os.path.join("third_party", "abseil-cpp"), + os.path.join("third_party", "protobuf"), os.path.join("third_party", "protobuf", "src"), + os.path.join("third_party", "protobuf", "upb"), os.path.join("third_party", "protobuf", "third_party", "utf8_range"), ] @@ -88,6 +90,11 @@ COPY_FILES_SOURCE_TARGET_PAIRS = [ ("src/compiler", "grpc_root/src/compiler"), ("third_party/abseil-cpp/absl", "third_party/abseil-cpp/absl"), ("third_party/protobuf/src", "third_party/protobuf/src"), + ("third_party/protobuf/upb", "third_party/protobuf/upb"), + ( + "third_party/protobuf/upb_generator", + "third_party/protobuf/upb_generator", + ), ( "third_party/protobuf/third_party/utf8_range", "third_party/protobuf/third_party/utf8_range", @@ -181,7 +188,7 @@ def _generate_deps_file_content(): # Collect .cc files (that will be later included in the native extension build) cc_files = [] for name in cc_files_output: - if name.endswith(".cc"): + if name.endswith(".c") or name.endswith(".cc"): filepath = _bazel_name_to_file_path(name) if filepath: cc_files.append(filepath) From ed8af1fb824e79faddf4824ef8a78d330d5a9425 Mon Sep 17 00:00:00 2001 From: nipil Date: Fri, 8 Mar 2024 11:39:34 -0800 Subject: [PATCH 22/52] [examples] fix binary path in systemd socket activation test script (#36034) Fix failing copy operations due to incorrect path levels Closes #36034 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/36034 from nipil:fix_systemd_socket_activation_path 2415112c22e2771d3cc4a4a3816ca6256fe462ff PiperOrigin-RevId: 613997786 --- examples/cpp/systemd_socket_activation/test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/cpp/systemd_socket_activation/test.sh b/examples/cpp/systemd_socket_activation/test.sh index f7cd42ebc64..607441bc135 100755 --- a/examples/cpp/systemd_socket_activation/test.sh +++ b/examples/cpp/systemd_socket_activation/test.sh @@ -37,8 +37,8 @@ pass() { } bazel build --define=use_systemd=true //examples/cpp/systemd_socket_activation:all || fail "Failed to build sd_sock_act" -cp ../../../../bazel-bin/examples/cpp/systemd_socket_activation/server /tmp/greeter_server -cp ../../../../bazel-bin/examples/cpp/systemd_socket_activation/client /tmp/greeter_client +cp ../../../bazel-bin/examples/cpp/systemd_socket_activation/server /tmp/greeter_server +cp ../../../bazel-bin/examples/cpp/systemd_socket_activation/client /tmp/greeter_client cat << EOF > /etc/systemd/system/sdsockact.service [Service] From b78f64569a136d65322cb7c2afd4f5d4250e7ad9 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Fri, 8 Mar 2024 12:46:18 -0800 Subject: [PATCH 23/52] [CI] Updated by review (#36083) Updated by review & Added missing import. Closes #36083 PiperOrigin-RevId: 614017839 --- src/python/grpcio/commands.py | 4 ++-- tools/distrib/python/grpcio_tools/setup.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py index 98324c955f9..0ceb0546a99 100644 --- a/src/python/grpcio/commands.py +++ b/src/python/grpcio/commands.py @@ -284,11 +284,11 @@ class BuildExt(build_ext.build_ext): def new_compile(obj, src, ext, cc_args, extra_postargs, pp_opts): if src.endswith(".c"): extra_postargs = [ - arg for arg in extra_postargs if not "-std=c++" in arg + arg for arg in extra_postargs if "-std=c++" not in arg ] elif src.endswith(".cc") or src.endswith(".cpp"): extra_postargs = [ - arg for arg in extra_postargs if not "-std=gnu99" in arg + arg for arg in extra_postargs if "-std=gnu99" not in arg ] return old_compile( obj, src, ext, cc_args, extra_postargs, pp_opts diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py index 011d43ce6f0..b778bc31ada 100644 --- a/tools/distrib/python/grpcio_tools/setup.py +++ b/tools/distrib/python/grpcio_tools/setup.py @@ -23,6 +23,7 @@ import subprocess from subprocess import PIPE import sys import sysconfig +import traceback import setuptools from setuptools import Extension @@ -141,11 +142,11 @@ class BuildExt(build_ext.build_ext): def new_compile(obj, src, ext, cc_args, extra_postargs, pp_opts): if src.endswith(".c"): extra_postargs = [ - arg for arg in extra_postargs if not "-std=c++" in arg + arg for arg in extra_postargs if "-std=c++" not in arg ] elif src.endswith(".cc") or src.endswith(".cpp"): extra_postargs = [ - arg for arg in extra_postargs if not "-std=c11" in arg + arg for arg in extra_postargs if "-std=c11" not in arg ] return old_compile(obj, src, ext, cc_args, extra_postargs, pp_opts) From 489d24cf7e970a2f3c7bac42d99371e90611c7c8 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Fri, 8 Mar 2024 17:18:10 -0800 Subject: [PATCH 24/52] [ruby] Make `GRPC::DeadlineExceeded` report properly (#33565) In grpc v1.46.2 and later versions, #29155 caused the Ruby client to return `GRPC::Core::CallError`, a non-standard error, with a message: `grpc_call_start_batch failed with outstanding read or write present (code=8)`. However, the actual error should have been `GRPC::DeadlineExceeded`. This occurred because when `GRPC::DeadlineExceeded` is raised, the `@metadata_received` flag doesn't get flipped. When `receive_and_check_status` runs to determine the error, the `RECV_INITIAL_METADATA` is erroneously sent again. To avoid this, flip the flag in an `ensure` block whenever the `RECV_INITIAL_METADATA` op is set. Closes #33283 Closes #33565 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/33565 from stanhu:sh-ruby-fix-issue-33283 850889558cf070e5e7b6eb471d71a61300b3442f PiperOrigin-RevId: 614088694 --- src/ruby/lib/grpc/generic/active_call.rb | 10 ++++++++-- src/ruby/spec/generic/rpc_server_spec.rb | 25 ++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/src/ruby/lib/grpc/generic/active_call.rb b/src/ruby/lib/grpc/generic/active_call.rb index 0c897d51998..d1d5bc7cbbd 100644 --- a/src/ruby/lib/grpc/generic/active_call.rb +++ b/src/ruby/lib/grpc/generic/active_call.rb @@ -169,10 +169,13 @@ module GRPC batch_result = @call.run_batch(ops) unless @metadata_received @call.metadata = batch_result.metadata - @metadata_received = true end set_input_stream_done attach_status_results_and_complete_call(batch_result) + ensure + # Ensure we don't attempt to request the initial metadata again + # in case an exception occurs. + @metadata_received = true end def attach_status_results_and_complete_call(recv_status_batch_result) @@ -258,12 +261,15 @@ module GRPC batch_result = @call.run_batch(ops) unless @metadata_received @call.metadata = batch_result.metadata - @metadata_received = true end get_message_from_batch_result(batch_result) rescue GRPC::Core::CallError => e GRPC.logger.info("remote_read: #{e}") nil + ensure + # Ensure we don't attempt to request the initial metadata again + # in case an exception occurs. + @metadata_received = true end def get_message_from_batch_result(recv_message_batch_result) diff --git a/src/ruby/spec/generic/rpc_server_spec.rb b/src/ruby/spec/generic/rpc_server_spec.rb index 6cb4ac2bda8..09b464c432f 100644 --- a/src/ruby/spec/generic/rpc_server_spec.rb +++ b/src/ruby/spec/generic/rpc_server_spec.rb @@ -61,6 +61,7 @@ FailingStub = FailingService.rpc_stub_class class SlowService include GRPC::GenericService rpc :an_rpc, EchoMsg, EchoMsg + rpc :a_server_streaming_rpc, EchoMsg, stream(EchoMsg) attr_reader :received_md, :delay def initialize(_default_var = 'ignored') @@ -74,6 +75,13 @@ class SlowService @received_md << call.metadata unless call.metadata.nil? req # send back the req as the response end + + def a_server_streaming_rpc(_, call) + GRPC.logger.info("starting a slow #{@delay} server streaming rpc") + sleep @delay + @received_md << call.metadata unless call.metadata.nil? + [EchoMsg.new, EchoMsg.new] + end end SlowStub = SlowService.rpc_stub_class @@ -410,6 +418,23 @@ describe GRPC::RpcServer do t.join end + it 'should raise DeadlineExceeded', server: true do + service = SlowService.new + @srv.handle(service) + t = Thread.new { @srv.run } + @srv.wait_till_running + req = EchoMsg.new + stub = SlowStub.new(@host, :this_channel_is_insecure, **client_opts) + timeout = service.delay - 0.1 + deadline = GRPC::Core::TimeConsts.from_relative_time(timeout) + responses = stub.a_server_streaming_rpc(req, + deadline: deadline, + metadata: { k1: 'v1', k2: 'v2' }) + expect { responses.to_a }.to raise_error(GRPC::DeadlineExceeded) + @srv.stop + t.join + end + it 'should handle cancellation correctly', server: true do request_received = false request_received_mu = Mutex.new From f8cfe1c16dc268ea4a8054851367a6deaa613a2a Mon Sep 17 00:00:00 2001 From: Hannah Shi Date: Fri, 8 Mar 2024 17:18:38 -0800 Subject: [PATCH 25/52] [ObjC] add boringssl swift package (#36062) gRPC swift package depends on [boringSSL-SwiftPM](https://github.com/firebase/boringSSL-SwiftPM), which hasn't been updated since more than two years ago (https://github.com/firebase/boringSSL-SwiftPM/pull/8). Updating it is not trivial as the `Package.swift` file is maintained manually, a recent [upgrade to swift 5.5](https://github.com/firebase/boringSSL-SwiftPM/pull/16) took good amount of time only to repeatedly exclude one (sometimes two) non-code files and retry build to get new errors, until the build succeeds. This PR uses the gRPC build system to generate the swift package file from the same boringssl version in third_party. Will use the output to update the boringSSL-SwiftPM repo after this PR is merged. Tested with ``` cp BoringSSL-Package.swift Package.swift swift build --target openssl_grpc swift test ``` Closes #36062 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/36062 from HannahShiSFB:add-boringssl-swift-package ac3e525bf2e06342a44b2bba618c91b727f1209a PiperOrigin-RevId: 614088802 --- BoringSSL-Package.swift | 319 +++++++++++++++++++++ build_handwritten.yaml | 3 + templates/BoringSSL-Package.swift.template | 64 +++++ test/boringssl_spm_build/test.cc | 34 +++ 4 files changed, 420 insertions(+) create mode 100644 BoringSSL-Package.swift create mode 100644 templates/BoringSSL-Package.swift.template create mode 100644 test/boringssl_spm_build/test.cc diff --git a/BoringSSL-Package.swift b/BoringSSL-Package.swift new file mode 100644 index 00000000000..363c62433ef --- /dev/null +++ b/BoringSSL-Package.swift @@ -0,0 +1,319 @@ +// swift-tools-version:5.5 +// The swift-tools-version declares the minimum version of Swift required to build this package. +import PackageDescription +import Foundation + +let basePath = "third_party/boringssl-with-bazel" +let privacyInfoPath = "../../src/objective-c/PrivacyInfo.xcprivacy" +let testPath = "test/boringssl_spm_build" + + + +let package = Package( + name: "BoringSSL-GRPC", + products: [ + .library( + name: "openssl_grpc", + targets: ["openssl_grpc"] + ) + ], + + targets: [ + .target( + name: "openssl_grpc", + path: basePath, + exclude: [ + ], + + sources: [ + "err_data.c", + "src/crypto/asn1/a_bitstr.c", + "src/crypto/asn1/a_bool.c", + "src/crypto/asn1/a_d2i_fp.c", + "src/crypto/asn1/a_dup.c", + "src/crypto/asn1/a_gentm.c", + "src/crypto/asn1/a_i2d_fp.c", + "src/crypto/asn1/a_int.c", + "src/crypto/asn1/a_mbstr.c", + "src/crypto/asn1/a_object.c", + "src/crypto/asn1/a_octet.c", + "src/crypto/asn1/a_strex.c", + "src/crypto/asn1/a_strnid.c", + "src/crypto/asn1/a_time.c", + "src/crypto/asn1/a_type.c", + "src/crypto/asn1/a_utctm.c", + "src/crypto/asn1/asn1_lib.c", + "src/crypto/asn1/asn1_par.c", + "src/crypto/asn1/asn_pack.c", + "src/crypto/asn1/f_int.c", + "src/crypto/asn1/f_string.c", + "src/crypto/asn1/posix_time.c", + "src/crypto/asn1/tasn_dec.c", + "src/crypto/asn1/tasn_enc.c", + "src/crypto/asn1/tasn_fre.c", + "src/crypto/asn1/tasn_new.c", + "src/crypto/asn1/tasn_typ.c", + "src/crypto/asn1/tasn_utl.c", + "src/crypto/base64/base64.c", + "src/crypto/bio/bio.c", + "src/crypto/bio/bio_mem.c", + "src/crypto/bio/connect.c", + "src/crypto/bio/errno.c", + "src/crypto/bio/fd.c", + "src/crypto/bio/file.c", + "src/crypto/bio/hexdump.c", + "src/crypto/bio/pair.c", + "src/crypto/bio/printf.c", + "src/crypto/bio/socket.c", + "src/crypto/bio/socket_helper.c", + "src/crypto/blake2/blake2.c", + "src/crypto/bn_extra/bn_asn1.c", + "src/crypto/bn_extra/convert.c", + "src/crypto/buf/buf.c", + "src/crypto/bytestring/asn1_compat.c", + "src/crypto/bytestring/ber.c", + "src/crypto/bytestring/cbb.c", + "src/crypto/bytestring/cbs.c", + "src/crypto/bytestring/unicode.c", + "src/crypto/chacha/chacha.c", + "src/crypto/cipher_extra/cipher_extra.c", + "src/crypto/cipher_extra/derive_key.c", + "src/crypto/cipher_extra/e_aesctrhmac.c", + "src/crypto/cipher_extra/e_aesgcmsiv.c", + "src/crypto/cipher_extra/e_chacha20poly1305.c", + "src/crypto/cipher_extra/e_des.c", + "src/crypto/cipher_extra/e_null.c", + "src/crypto/cipher_extra/e_rc2.c", + "src/crypto/cipher_extra/e_rc4.c", + "src/crypto/cipher_extra/e_tls.c", + "src/crypto/cipher_extra/tls_cbc.c", + "src/crypto/conf/conf.c", + "src/crypto/cpu_aarch64_apple.c", + "src/crypto/cpu_aarch64_fuchsia.c", + "src/crypto/cpu_aarch64_linux.c", + "src/crypto/cpu_aarch64_openbsd.c", + "src/crypto/cpu_aarch64_sysreg.c", + "src/crypto/cpu_aarch64_win.c", + "src/crypto/cpu_arm_freebsd.c", + "src/crypto/cpu_arm_linux.c", + "src/crypto/cpu_intel.c", + "src/crypto/crypto.c", + "src/crypto/curve25519/curve25519.c", + "src/crypto/curve25519/curve25519_64_adx.c", + "src/crypto/curve25519/spake25519.c", + "src/crypto/des/des.c", + "src/crypto/dh_extra/dh_asn1.c", + "src/crypto/dh_extra/params.c", + "src/crypto/digest_extra/digest_extra.c", + "src/crypto/dsa/dsa.c", + "src/crypto/dsa/dsa_asn1.c", + "src/crypto/ec_extra/ec_asn1.c", + "src/crypto/ec_extra/ec_derive.c", + "src/crypto/ec_extra/hash_to_curve.c", + "src/crypto/ecdh_extra/ecdh_extra.c", + "src/crypto/ecdsa_extra/ecdsa_asn1.c", + "src/crypto/engine/engine.c", + "src/crypto/err/err.c", + "src/crypto/evp/evp.c", + "src/crypto/evp/evp_asn1.c", + "src/crypto/evp/evp_ctx.c", + "src/crypto/evp/p_dsa_asn1.c", + "src/crypto/evp/p_ec.c", + "src/crypto/evp/p_ec_asn1.c", + "src/crypto/evp/p_ed25519.c", + "src/crypto/evp/p_ed25519_asn1.c", + "src/crypto/evp/p_hkdf.c", + "src/crypto/evp/p_rsa.c", + "src/crypto/evp/p_rsa_asn1.c", + "src/crypto/evp/p_x25519.c", + "src/crypto/evp/p_x25519_asn1.c", + "src/crypto/evp/pbkdf.c", + "src/crypto/evp/print.c", + "src/crypto/evp/scrypt.c", + "src/crypto/evp/sign.c", + "src/crypto/ex_data.c", + "src/crypto/fipsmodule/bcm.c", + "src/crypto/fipsmodule/fips_shared_support.c", + "src/crypto/hpke/hpke.c", + "src/crypto/hrss/hrss.c", + "src/crypto/keccak/keccak.c", + "src/crypto/kyber/kyber.c", + "src/crypto/lhash/lhash.c", + "src/crypto/mem.c", + "src/crypto/obj/obj.c", + "src/crypto/obj/obj_xref.c", + "src/crypto/pem/pem_all.c", + "src/crypto/pem/pem_info.c", + "src/crypto/pem/pem_lib.c", + "src/crypto/pem/pem_oth.c", + "src/crypto/pem/pem_pk8.c", + "src/crypto/pem/pem_pkey.c", + "src/crypto/pem/pem_x509.c", + "src/crypto/pem/pem_xaux.c", + "src/crypto/pkcs7/pkcs7.c", + "src/crypto/pkcs7/pkcs7_x509.c", + "src/crypto/pkcs8/p5_pbev2.c", + "src/crypto/pkcs8/pkcs8.c", + "src/crypto/pkcs8/pkcs8_x509.c", + "src/crypto/poly1305/poly1305.c", + "src/crypto/poly1305/poly1305_arm.c", + "src/crypto/poly1305/poly1305_vec.c", + "src/crypto/pool/pool.c", + "src/crypto/rand_extra/deterministic.c", + "src/crypto/rand_extra/forkunsafe.c", + "src/crypto/rand_extra/getentropy.c", + "src/crypto/rand_extra/ios.c", + "src/crypto/rand_extra/passive.c", + "src/crypto/rand_extra/rand_extra.c", + "src/crypto/rand_extra/trusty.c", + "src/crypto/rand_extra/windows.c", + "src/crypto/rc4/rc4.c", + "src/crypto/refcount.c", + "src/crypto/rsa_extra/rsa_asn1.c", + "src/crypto/rsa_extra/rsa_crypt.c", + "src/crypto/rsa_extra/rsa_print.c", + "src/crypto/siphash/siphash.c", + "src/crypto/spx/address.c", + "src/crypto/spx/fors.c", + "src/crypto/spx/merkle.c", + "src/crypto/spx/spx.c", + "src/crypto/spx/spx_util.c", + "src/crypto/spx/thash.c", + "src/crypto/spx/wots.c", + "src/crypto/stack/stack.c", + "src/crypto/thread.c", + "src/crypto/thread_none.c", + "src/crypto/thread_pthread.c", + "src/crypto/thread_win.c", + "src/crypto/trust_token/pmbtoken.c", + "src/crypto/trust_token/trust_token.c", + "src/crypto/trust_token/voprf.c", + "src/crypto/x509/a_digest.c", + "src/crypto/x509/a_sign.c", + "src/crypto/x509/a_verify.c", + "src/crypto/x509/algorithm.c", + "src/crypto/x509/asn1_gen.c", + "src/crypto/x509/by_dir.c", + "src/crypto/x509/by_file.c", + "src/crypto/x509/i2d_pr.c", + "src/crypto/x509/name_print.c", + "src/crypto/x509/policy.c", + "src/crypto/x509/rsa_pss.c", + "src/crypto/x509/t_crl.c", + "src/crypto/x509/t_req.c", + "src/crypto/x509/t_x509.c", + "src/crypto/x509/t_x509a.c", + "src/crypto/x509/v3_akey.c", + "src/crypto/x509/v3_akeya.c", + "src/crypto/x509/v3_alt.c", + "src/crypto/x509/v3_bcons.c", + "src/crypto/x509/v3_bitst.c", + "src/crypto/x509/v3_conf.c", + "src/crypto/x509/v3_cpols.c", + "src/crypto/x509/v3_crld.c", + "src/crypto/x509/v3_enum.c", + "src/crypto/x509/v3_extku.c", + "src/crypto/x509/v3_genn.c", + "src/crypto/x509/v3_ia5.c", + "src/crypto/x509/v3_info.c", + "src/crypto/x509/v3_int.c", + "src/crypto/x509/v3_lib.c", + "src/crypto/x509/v3_ncons.c", + "src/crypto/x509/v3_ocsp.c", + "src/crypto/x509/v3_pcons.c", + "src/crypto/x509/v3_pmaps.c", + "src/crypto/x509/v3_prn.c", + "src/crypto/x509/v3_purp.c", + "src/crypto/x509/v3_skey.c", + "src/crypto/x509/v3_utl.c", + "src/crypto/x509/x509.c", + "src/crypto/x509/x509_att.c", + "src/crypto/x509/x509_cmp.c", + "src/crypto/x509/x509_d2.c", + "src/crypto/x509/x509_def.c", + "src/crypto/x509/x509_ext.c", + "src/crypto/x509/x509_lu.c", + "src/crypto/x509/x509_obj.c", + "src/crypto/x509/x509_req.c", + "src/crypto/x509/x509_set.c", + "src/crypto/x509/x509_trs.c", + "src/crypto/x509/x509_txt.c", + "src/crypto/x509/x509_v3.c", + "src/crypto/x509/x509_vfy.c", + "src/crypto/x509/x509_vpm.c", + "src/crypto/x509/x509cset.c", + "src/crypto/x509/x509name.c", + "src/crypto/x509/x509rset.c", + "src/crypto/x509/x509spki.c", + "src/crypto/x509/x_algor.c", + "src/crypto/x509/x_all.c", + "src/crypto/x509/x_attrib.c", + "src/crypto/x509/x_crl.c", + "src/crypto/x509/x_exten.c", + "src/crypto/x509/x_name.c", + "src/crypto/x509/x_pubkey.c", + "src/crypto/x509/x_req.c", + "src/crypto/x509/x_sig.c", + "src/crypto/x509/x_spki.c", + "src/crypto/x509/x_val.c", + "src/crypto/x509/x_x509.c", + "src/crypto/x509/x_x509a.c", + "src/ssl/bio_ssl.cc", + "src/ssl/d1_both.cc", + "src/ssl/d1_lib.cc", + "src/ssl/d1_pkt.cc", + "src/ssl/d1_srtp.cc", + "src/ssl/dtls_method.cc", + "src/ssl/dtls_record.cc", + "src/ssl/encrypted_client_hello.cc", + "src/ssl/extensions.cc", + "src/ssl/handoff.cc", + "src/ssl/handshake.cc", + "src/ssl/handshake_client.cc", + "src/ssl/handshake_server.cc", + "src/ssl/s3_both.cc", + "src/ssl/s3_lib.cc", + "src/ssl/s3_pkt.cc", + "src/ssl/ssl_aead_ctx.cc", + "src/ssl/ssl_asn1.cc", + "src/ssl/ssl_buffer.cc", + "src/ssl/ssl_cert.cc", + "src/ssl/ssl_cipher.cc", + "src/ssl/ssl_file.cc", + "src/ssl/ssl_key_share.cc", + "src/ssl/ssl_lib.cc", + "src/ssl/ssl_privkey.cc", + "src/ssl/ssl_session.cc", + "src/ssl/ssl_stat.cc", + "src/ssl/ssl_transcript.cc", + "src/ssl/ssl_versions.cc", + "src/ssl/ssl_x509.cc", + "src/ssl/t1_enc.cc", + "src/ssl/tls13_both.cc", + "src/ssl/tls13_client.cc", + "src/ssl/tls13_enc.cc", + "src/ssl/tls13_server.cc", + "src/ssl/tls_method.cc", + "src/ssl/tls_record.cc", + ], + resources: [ + .copy(privacyInfoPath), + ], + publicHeadersPath: "src/include", + + cSettings: [ + .define("OPENSSL_NO_ASM", to: "1"), + .headerSearchPath("./"), + .headerSearchPath("include/"), + ] + ), + .testTarget( + name: "build-test", + dependencies: [ + "openssl_grpc", + ], + path: testPath + ), + ], + cxxLanguageStandard: .cxx14 +) diff --git a/build_handwritten.yaml b/build_handwritten.yaml index 82e9226e270..15c9a8de6ce 100644 --- a/build_handwritten.yaml +++ b/build_handwritten.yaml @@ -199,6 +199,9 @@ ruby_gem: - boringssl - re2 - z +swift_boringssl_package: + deps: + - boringssl swift_package: deps: - grpc diff --git a/templates/BoringSSL-Package.swift.template b/templates/BoringSSL-Package.swift.template new file mode 100644 index 00000000000..519d77dfba1 --- /dev/null +++ b/templates/BoringSSL-Package.swift.template @@ -0,0 +1,64 @@ +%YAML 1.2 +--- | + // swift-tools-version:5.5 + // The swift-tools-version declares the minimum version of Swift required to build this package. + import PackageDescription + import Foundation + + let basePath = "third_party/boringssl-with-bazel" + let privacyInfoPath = "../../src/objective-c/PrivacyInfo.xcprivacy" + let testPath = "test/boringssl_spm_build" + + + + let package = Package( + name: "BoringSSL-GRPC", + products: [ + .library( + name: "openssl_grpc", + targets: ["openssl_grpc"] + ) + ], + + targets: [ + .target( + name: "openssl_grpc", + path: basePath, + exclude: [ + ], + <% + def remove_prefix(s, prefix): + return s[len(prefix):] if s.startswith(prefix) else s + files = [] + lib_maps = {lib.name: lib for lib in libs} + for dep in swift_boringssl_package.get('deps', []): + lib = lib_maps[dep] + files.extend(lib.src) + files = sorted(set({remove_prefix(file, "third_party/boringssl-with-bazel/"): file for file in files})) + %> + sources: [ + % for file in files: + "${file}", + % endfor + ], + resources: [ + .copy(privacyInfoPath), + ], + publicHeadersPath: "src/include", + + cSettings: [ + .define("OPENSSL_NO_ASM", to: "1"), + .headerSearchPath("./"), + .headerSearchPath("include/"), + ] + ), + .testTarget( + name: "build-test", + dependencies: [ + "openssl_grpc", + ], + path: testPath + ), + ], + cxxLanguageStandard: .cxx14 + ) diff --git a/test/boringssl_spm_build/test.cc b/test/boringssl_spm_build/test.cc new file mode 100644 index 00000000000..5bdbcebc39d --- /dev/null +++ b/test/boringssl_spm_build/test.cc @@ -0,0 +1,34 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Test for duplicate `_main` symbol. +int main(int argc, char** argv) {} From 7533328075e3cccb2aab7a2a1e990d7eb3a4f984 Mon Sep 17 00:00:00 2001 From: Vignesh Babu Date: Mon, 11 Mar 2024 11:14:30 -0700 Subject: [PATCH 26/52] [server] Allow configuring max incoming connections at the server (#36088) Closes #36088 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/36088 from Vignesh2208:server-fix e62a8f7ae729cfe8c7c6806d5c7e5c8c3866c2ee PiperOrigin-RevId: 614729878 --- CMakeLists.txt | 8 ++ Makefile | 1 + Package.swift | 2 + build_autogenerated.yaml | 16 ++++ config.m4 | 1 + config.w32 | 1 + gRPC-C++.podspec | 2 + gRPC-Core.podspec | 3 + grpc.gemspec | 2 + grpc.gyp | 3 + include/grpc/impl/channel_arg_names.h | 4 + package.xml | 2 + src/core/BUILD | 19 +++++ .../transport/chttp2/server/chttp2_server.cc | 32 +++++++- .../lib/resource_quota/connection_quota.cc | 71 ++++++++++++++++++ .../lib/resource_quota/connection_quota.h | 61 +++++++++++++++ src/python/grpcio/grpc_core_dependencies.py | 1 + .../resource_quota_end2end_stress_test.cc | 74 ++++++++++++++++++- tools/doxygen/Doxyfile.c++.internal | 2 + tools/doxygen/Doxyfile.core.internal | 2 + 20 files changed, 305 insertions(+), 2 deletions(-) create mode 100644 src/core/lib/resource_quota/connection_quota.cc create mode 100644 src/core/lib/resource_quota/connection_quota.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 8faf34698dd..d5fc01ad30c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2383,6 +2383,7 @@ add_library(grpc src/core/lib/promise/trace.cc src/core/lib/resource_quota/api.cc src/core/lib/resource_quota/arena.cc + src/core/lib/resource_quota/connection_quota.cc src/core/lib/resource_quota/memory_quota.cc src/core/lib/resource_quota/periodic_update.cc src/core/lib/resource_quota/resource_quota.cc @@ -3145,6 +3146,7 @@ add_library(grpc_unsecure src/core/lib/promise/trace.cc src/core/lib/resource_quota/api.cc src/core/lib/resource_quota/arena.cc + src/core/lib/resource_quota/connection_quota.cc src/core/lib/resource_quota/memory_quota.cc src/core/lib/resource_quota/periodic_update.cc src/core/lib/resource_quota/resource_quota.cc @@ -5252,6 +5254,7 @@ add_library(grpc_authorization_provider src/core/lib/promise/trace.cc src/core/lib/resource_quota/api.cc src/core/lib/resource_quota/arena.cc + src/core/lib/resource_quota/connection_quota.cc src/core/lib/resource_quota/memory_quota.cc src/core/lib/resource_quota/periodic_update.cc src/core/lib/resource_quota/resource_quota.cc @@ -10177,6 +10180,7 @@ add_executable(chunked_vector_test src/core/lib/promise/activity.cc src/core/lib/promise/trace.cc src/core/lib/resource_quota/arena.cc + src/core/lib/resource_quota/connection_quota.cc src/core/lib/resource_quota/memory_quota.cc src/core/lib/resource_quota/periodic_update.cc src/core/lib/resource_quota/resource_quota.cc @@ -14175,6 +14179,7 @@ add_executable(flow_control_test src/core/lib/iomgr/iomgr_internal.cc src/core/lib/promise/activity.cc src/core/lib/promise/trace.cc + src/core/lib/resource_quota/connection_quota.cc src/core/lib/resource_quota/memory_quota.cc src/core/lib/resource_quota/periodic_update.cc src/core/lib/resource_quota/resource_quota.cc @@ -14263,6 +14268,7 @@ add_executable(for_each_test src/core/lib/promise/activity.cc src/core/lib/promise/trace.cc src/core/lib/resource_quota/arena.cc + src/core/lib/resource_quota/connection_quota.cc src/core/lib/resource_quota/memory_quota.cc src/core/lib/resource_quota/periodic_update.cc src/core/lib/resource_quota/resource_quota.cc @@ -17709,6 +17715,7 @@ add_executable(interceptor_list_test src/core/lib/promise/activity.cc src/core/lib/promise/trace.cc src/core/lib/resource_quota/arena.cc + src/core/lib/resource_quota/connection_quota.cc src/core/lib/resource_quota/memory_quota.cc src/core/lib/resource_quota/periodic_update.cc src/core/lib/resource_quota/resource_quota.cc @@ -18900,6 +18907,7 @@ add_executable(map_pipe_test src/core/lib/promise/activity.cc src/core/lib/promise/trace.cc src/core/lib/resource_quota/arena.cc + src/core/lib/resource_quota/connection_quota.cc src/core/lib/resource_quota/memory_quota.cc src/core/lib/resource_quota/periodic_update.cc src/core/lib/resource_quota/resource_quota.cc diff --git a/Makefile b/Makefile index 4bdbbec686f..23f195b095a 100644 --- a/Makefile +++ b/Makefile @@ -1309,6 +1309,7 @@ LIBGRPC_SRC = \ src/core/lib/promise/trace.cc \ src/core/lib/resource_quota/api.cc \ src/core/lib/resource_quota/arena.cc \ + src/core/lib/resource_quota/connection_quota.cc \ src/core/lib/resource_quota/memory_quota.cc \ src/core/lib/resource_quota/periodic_update.cc \ src/core/lib/resource_quota/resource_quota.cc \ diff --git a/Package.swift b/Package.swift index 64d6d866bf7..9737494d3bc 100644 --- a/Package.swift +++ b/Package.swift @@ -1608,6 +1608,8 @@ let package = Package( "src/core/lib/resource_quota/api.h", "src/core/lib/resource_quota/arena.cc", "src/core/lib/resource_quota/arena.h", + "src/core/lib/resource_quota/connection_quota.cc", + "src/core/lib/resource_quota/connection_quota.h", "src/core/lib/resource_quota/memory_quota.cc", "src/core/lib/resource_quota/memory_quota.h", "src/core/lib/resource_quota/periodic_update.cc", diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index 967ad017f7d..e9478a2a4b8 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -1041,6 +1041,7 @@ libs: - src/core/lib/promise/try_seq.h - src/core/lib/resource_quota/api.h - src/core/lib/resource_quota/arena.h + - src/core/lib/resource_quota/connection_quota.h - src/core/lib/resource_quota/memory_quota.h - src/core/lib/resource_quota/periodic_update.h - src/core/lib/resource_quota/resource_quota.h @@ -1837,6 +1838,7 @@ libs: - src/core/lib/promise/trace.cc - src/core/lib/resource_quota/api.cc - src/core/lib/resource_quota/arena.cc + - src/core/lib/resource_quota/connection_quota.cc - src/core/lib/resource_quota/memory_quota.cc - src/core/lib/resource_quota/periodic_update.cc - src/core/lib/resource_quota/resource_quota.cc @@ -2542,6 +2544,7 @@ libs: - src/core/lib/promise/try_seq.h - src/core/lib/resource_quota/api.h - src/core/lib/resource_quota/arena.h + - src/core/lib/resource_quota/connection_quota.h - src/core/lib/resource_quota/memory_quota.h - src/core/lib/resource_quota/periodic_update.h - src/core/lib/resource_quota/resource_quota.h @@ -2961,6 +2964,7 @@ libs: - src/core/lib/promise/trace.cc - src/core/lib/resource_quota/api.cc - src/core/lib/resource_quota/arena.cc + - src/core/lib/resource_quota/connection_quota.cc - src/core/lib/resource_quota/memory_quota.cc - src/core/lib/resource_quota/periodic_update.cc - src/core/lib/resource_quota/resource_quota.cc @@ -4619,6 +4623,7 @@ libs: - src/core/lib/promise/try_seq.h - src/core/lib/resource_quota/api.h - src/core/lib/resource_quota/arena.h + - src/core/lib/resource_quota/connection_quota.h - src/core/lib/resource_quota/memory_quota.h - src/core/lib/resource_quota/periodic_update.h - src/core/lib/resource_quota/resource_quota.h @@ -4916,6 +4921,7 @@ libs: - src/core/lib/promise/trace.cc - src/core/lib/resource_quota/api.cc - src/core/lib/resource_quota/arena.cc + - src/core/lib/resource_quota/connection_quota.cc - src/core/lib/resource_quota/memory_quota.cc - src/core/lib/resource_quota/periodic_update.cc - src/core/lib/resource_quota/resource_quota.cc @@ -7689,6 +7695,7 @@ targets: - src/core/lib/promise/seq.h - src/core/lib/promise/trace.h - src/core/lib/resource_quota/arena.h + - src/core/lib/resource_quota/connection_quota.h - src/core/lib/resource_quota/memory_quota.h - src/core/lib/resource_quota/periodic_update.h - src/core/lib/resource_quota/resource_quota.h @@ -7738,6 +7745,7 @@ targets: - src/core/lib/promise/activity.cc - src/core/lib/promise/trace.cc - src/core/lib/resource_quota/arena.cc + - src/core/lib/resource_quota/connection_quota.cc - src/core/lib/resource_quota/memory_quota.cc - src/core/lib/resource_quota/periodic_update.cc - src/core/lib/resource_quota/resource_quota.cc @@ -9785,6 +9793,7 @@ targets: - src/core/lib/promise/race.h - src/core/lib/promise/seq.h - src/core/lib/promise/trace.h + - src/core/lib/resource_quota/connection_quota.h - src/core/lib/resource_quota/memory_quota.h - src/core/lib/resource_quota/periodic_update.h - src/core/lib/resource_quota/resource_quota.h @@ -9839,6 +9848,7 @@ targets: - src/core/lib/iomgr/iomgr_internal.cc - src/core/lib/promise/activity.cc - src/core/lib/promise/trace.cc + - src/core/lib/resource_quota/connection_quota.cc - src/core/lib/resource_quota/memory_quota.cc - src/core/lib/resource_quota/periodic_update.cc - src/core/lib/resource_quota/resource_quota.cc @@ -9928,6 +9938,7 @@ targets: - src/core/lib/promise/trace.h - src/core/lib/promise/try_seq.h - src/core/lib/resource_quota/arena.h + - src/core/lib/resource_quota/connection_quota.h - src/core/lib/resource_quota/memory_quota.h - src/core/lib/resource_quota/periodic_update.h - src/core/lib/resource_quota/resource_quota.h @@ -9978,6 +9989,7 @@ targets: - src/core/lib/promise/activity.cc - src/core/lib/promise/trace.cc - src/core/lib/resource_quota/arena.cc + - src/core/lib/resource_quota/connection_quota.cc - src/core/lib/resource_quota/memory_quota.cc - src/core/lib/resource_quota/periodic_update.cc - src/core/lib/resource_quota/resource_quota.cc @@ -11578,6 +11590,7 @@ targets: - src/core/lib/promise/seq.h - src/core/lib/promise/trace.h - src/core/lib/resource_quota/arena.h + - src/core/lib/resource_quota/connection_quota.h - src/core/lib/resource_quota/memory_quota.h - src/core/lib/resource_quota/periodic_update.h - src/core/lib/resource_quota/resource_quota.h @@ -11628,6 +11641,7 @@ targets: - src/core/lib/promise/activity.cc - src/core/lib/promise/trace.cc - src/core/lib/resource_quota/arena.cc + - src/core/lib/resource_quota/connection_quota.cc - src/core/lib/resource_quota/memory_quota.cc - src/core/lib/resource_quota/periodic_update.cc - src/core/lib/resource_quota/resource_quota.cc @@ -12256,6 +12270,7 @@ targets: - src/core/lib/promise/trace.h - src/core/lib/promise/try_seq.h - src/core/lib/resource_quota/arena.h + - src/core/lib/resource_quota/connection_quota.h - src/core/lib/resource_quota/memory_quota.h - src/core/lib/resource_quota/periodic_update.h - src/core/lib/resource_quota/resource_quota.h @@ -12306,6 +12321,7 @@ targets: - src/core/lib/promise/activity.cc - src/core/lib/promise/trace.cc - src/core/lib/resource_quota/arena.cc + - src/core/lib/resource_quota/connection_quota.cc - src/core/lib/resource_quota/memory_quota.cc - src/core/lib/resource_quota/periodic_update.cc - src/core/lib/resource_quota/resource_quota.cc diff --git a/config.m4 b/config.m4 index 4600f743302..71180188c82 100644 --- a/config.m4 +++ b/config.m4 @@ -684,6 +684,7 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/promise/trace.cc \ src/core/lib/resource_quota/api.cc \ src/core/lib/resource_quota/arena.cc \ + src/core/lib/resource_quota/connection_quota.cc \ src/core/lib/resource_quota/memory_quota.cc \ src/core/lib/resource_quota/periodic_update.cc \ src/core/lib/resource_quota/resource_quota.cc \ diff --git a/config.w32 b/config.w32 index dcf40dbb7d3..b9e6c920b2e 100644 --- a/config.w32 +++ b/config.w32 @@ -649,6 +649,7 @@ if (PHP_GRPC != "no") { "src\\core\\lib\\promise\\trace.cc " + "src\\core\\lib\\resource_quota\\api.cc " + "src\\core\\lib\\resource_quota\\arena.cc " + + "src\\core\\lib\\resource_quota\\connection_quota.cc " + "src\\core\\lib\\resource_quota\\memory_quota.cc " + "src\\core\\lib\\resource_quota\\periodic_update.cc " + "src\\core\\lib\\resource_quota\\resource_quota.cc " + diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index 84a7bfef691..bad4a958ef8 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -1147,6 +1147,7 @@ Pod::Spec.new do |s| 'src/core/lib/promise/try_seq.h', 'src/core/lib/resource_quota/api.h', 'src/core/lib/resource_quota/arena.h', + 'src/core/lib/resource_quota/connection_quota.h', 'src/core/lib/resource_quota/memory_quota.h', 'src/core/lib/resource_quota/periodic_update.h', 'src/core/lib/resource_quota/resource_quota.h', @@ -2410,6 +2411,7 @@ Pod::Spec.new do |s| 'src/core/lib/promise/try_seq.h', 'src/core/lib/resource_quota/api.h', 'src/core/lib/resource_quota/arena.h', + 'src/core/lib/resource_quota/connection_quota.h', 'src/core/lib/resource_quota/memory_quota.h', 'src/core/lib/resource_quota/periodic_update.h', 'src/core/lib/resource_quota/resource_quota.h', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 35adc20a350..ca1624e7ac2 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -1722,6 +1722,8 @@ Pod::Spec.new do |s| 'src/core/lib/resource_quota/api.h', 'src/core/lib/resource_quota/arena.cc', 'src/core/lib/resource_quota/arena.h', + 'src/core/lib/resource_quota/connection_quota.cc', + 'src/core/lib/resource_quota/connection_quota.h', 'src/core/lib/resource_quota/memory_quota.cc', 'src/core/lib/resource_quota/memory_quota.h', 'src/core/lib/resource_quota/periodic_update.cc', @@ -3191,6 +3193,7 @@ Pod::Spec.new do |s| 'src/core/lib/promise/try_seq.h', 'src/core/lib/resource_quota/api.h', 'src/core/lib/resource_quota/arena.h', + 'src/core/lib/resource_quota/connection_quota.h', 'src/core/lib/resource_quota/memory_quota.h', 'src/core/lib/resource_quota/periodic_update.h', 'src/core/lib/resource_quota/resource_quota.h', diff --git a/grpc.gemspec b/grpc.gemspec index eaed281aa80..12d4db8b99b 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -1614,6 +1614,8 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/resource_quota/api.h ) s.files += %w( src/core/lib/resource_quota/arena.cc ) s.files += %w( src/core/lib/resource_quota/arena.h ) + s.files += %w( src/core/lib/resource_quota/connection_quota.cc ) + s.files += %w( src/core/lib/resource_quota/connection_quota.h ) s.files += %w( src/core/lib/resource_quota/memory_quota.cc ) s.files += %w( src/core/lib/resource_quota/memory_quota.h ) s.files += %w( src/core/lib/resource_quota/periodic_update.cc ) diff --git a/grpc.gyp b/grpc.gyp index 1f7a89d8d52..df31e5f4f3b 100644 --- a/grpc.gyp +++ b/grpc.gyp @@ -883,6 +883,7 @@ 'src/core/lib/promise/trace.cc', 'src/core/lib/resource_quota/api.cc', 'src/core/lib/resource_quota/arena.cc', + 'src/core/lib/resource_quota/connection_quota.cc', 'src/core/lib/resource_quota/memory_quota.cc', 'src/core/lib/resource_quota/periodic_update.cc', 'src/core/lib/resource_quota/resource_quota.cc', @@ -1422,6 +1423,7 @@ 'src/core/lib/promise/trace.cc', 'src/core/lib/resource_quota/api.cc', 'src/core/lib/resource_quota/arena.cc', + 'src/core/lib/resource_quota/connection_quota.cc', 'src/core/lib/resource_quota/memory_quota.cc', 'src/core/lib/resource_quota/periodic_update.cc', 'src/core/lib/resource_quota/resource_quota.cc', @@ -2228,6 +2230,7 @@ 'src/core/lib/promise/trace.cc', 'src/core/lib/resource_quota/api.cc', 'src/core/lib/resource_quota/arena.cc', + 'src/core/lib/resource_quota/connection_quota.cc', 'src/core/lib/resource_quota/memory_quota.cc', 'src/core/lib/resource_quota/periodic_update.cc', 'src/core/lib/resource_quota/resource_quota.cc', diff --git a/include/grpc/impl/channel_arg_names.h b/include/grpc/impl/channel_arg_names.h index b42bf7026ef..7ba19bebb17 100644 --- a/include/grpc/impl/channel_arg_names.h +++ b/include/grpc/impl/channel_arg_names.h @@ -395,6 +395,10 @@ * factory. */ #define GRPC_ARG_EVENT_ENGINE_USE_MEMORY_ALLOCATOR_FACTORY \ "grpc.event_engine_use_memory_allocator_factory" +/** Configure the max number of allowed incoming connections to the server. + * If unspecified, it is unlimited */ +#define GRPC_ARG_MAX_ALLOWED_INCOMING_CONNECTIONS \ + "grpc.max_allowed_incoming_connections" /** \} */ #endif /* GRPC_IMPL_CHANNEL_ARG_NAMES_H */ diff --git a/package.xml b/package.xml index fa35a2c4033..1ba949ae253 100644 --- a/package.xml +++ b/package.xml @@ -1596,6 +1596,8 @@ + + diff --git a/src/core/BUILD b/src/core/BUILD index f0c58279faf..46708db643f 100644 --- a/src/core/BUILD +++ b/src/core/BUILD @@ -1399,6 +1399,23 @@ grpc_cc_library( ], ) +grpc_cc_library( + name = "connection_quota", + srcs = [ + "lib/resource_quota/connection_quota.cc", + ], + hdrs = [ + "lib/resource_quota/connection_quota.h", + ], + external_deps = ["absl/base:core_headers"], + deps = [ + "memory_quota", + "ref_counted", + "//:gpr", + "//:ref_counted_ptr", + ], +) + grpc_cc_library( name = "resource_quota_trace", srcs = [ @@ -1426,6 +1443,7 @@ grpc_cc_library( "@grpc:alt_grpc_base_legacy", ], deps = [ + "connection_quota", "memory_quota", "ref_counted", "thread_quota", @@ -6747,6 +6765,7 @@ grpc_cc_library( "channel_args", "channel_args_endpoint_config", "closure", + "connection_quota", "error", "error_utils", "grpc_insecure_credentials", diff --git a/src/core/ext/transport/chttp2/server/chttp2_server.cc b/src/core/ext/transport/chttp2/server/chttp2_server.cc index dea28445a4f..7ac39d01cff 100644 --- a/src/core/ext/transport/chttp2/server/chttp2_server.cc +++ b/src/core/ext/transport/chttp2/server/chttp2_server.cc @@ -72,6 +72,7 @@ #include "src/core/lib/iomgr/tcp_server.h" #include "src/core/lib/iomgr/unix_sockets_posix.h" #include "src/core/lib/iomgr/vsock.h" +#include "src/core/lib/resource_quota/connection_quota.h" #include "src/core/lib/resource_quota/memory_quota.h" #include "src/core/lib/resource_quota/resource_quota.h" #include "src/core/lib/security/credentials/credentials.h" @@ -283,6 +284,7 @@ class Chttp2ServerListener : public Server::ListenerInterface { grpc_closure* on_destroy_done_ ABSL_GUARDED_BY(mu_) = nullptr; RefCountedPtr channelz_listen_socket_; MemoryQuotaRefPtr memory_quota_; + ConnectionQuotaRefPtr connection_quota_; }; // @@ -504,6 +506,18 @@ void Chttp2ServerListener::ActiveConnection::HandshakingState::OnHandshakeDone( } else { // Remove the connection from the connections_ map since OnClose() // will not be invoked when a config fetcher is set. + auto connection_quota = + self->connection_->listener_->connection_quota_->Ref() + .release(); + auto on_close_transport = [](void* arg, + grpc_error_handle /*handle*/) { + ConnectionQuota* connection_quota = + static_cast(arg); + connection_quota->ReleaseConnections(1); + connection_quota->Unref(); + }; + on_close = GRPC_CLOSURE_CREATE(on_close_transport, connection_quota, + grpc_schedule_on_exec_ctx_); cleanup_connection = true; } grpc_chttp2_transport_start_reading(transport, args->read_buffer, @@ -658,6 +672,7 @@ void Chttp2ServerListener::ActiveConnection::OnClose( self->drain_grace_timer_handle_.reset(); } } + self->listener_->connection_quota_->ReleaseConnections(1); self->Unref(); } @@ -762,7 +777,14 @@ Chttp2ServerListener::Chttp2ServerListener( : server_(server), args_modifier_(args_modifier), args_(args), - memory_quota_(args.GetObject()->memory_quota()) { + memory_quota_(args.GetObject()->memory_quota()), + connection_quota_(MakeRefCounted()) { + auto max_allowed_incoming_connections = + args.GetInt(GRPC_ARG_MAX_ALLOWED_INCOMING_CONNECTIONS); + if (max_allowed_incoming_connections.has_value()) { + connection_quota_->SetMaxIncomingConnections( + max_allowed_incoming_connections.value()); + } GRPC_CLOSURE_INIT(&tcp_server_shutdown_complete_, TcpServerShutdownComplete, this, grpc_schedule_on_exec_ctx); } @@ -821,6 +843,14 @@ void Chttp2ServerListener::OnAccept(void* arg, grpc_endpoint* tcp, grpc_endpoint_destroy(tcp); gpr_free(acceptor); }; + if (!self->connection_quota_->AllowIncomingConnection( + self->memory_quota_, grpc_endpoint_get_peer(tcp))) { + grpc_error_handle error = GRPC_ERROR_CREATE( + "Rejected incoming connection because configured connection quota " + "limits have been exceeded."); + endpoint_cleanup(error); + return; + } if (self->server_->config_fetcher() != nullptr) { if (connection_manager == nullptr) { grpc_error_handle error = GRPC_ERROR_CREATE( diff --git a/src/core/lib/resource_quota/connection_quota.cc b/src/core/lib/resource_quota/connection_quota.cc new file mode 100644 index 00000000000..75d3c7b8edd --- /dev/null +++ b/src/core/lib/resource_quota/connection_quota.cc @@ -0,0 +1,71 @@ +// Copyright 2024 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "src/core/lib/resource_quota/connection_quota.h" + +#include +#include + +#include + +namespace grpc_core { + +ConnectionQuota::ConnectionQuota() = default; + +void ConnectionQuota::SetMaxIncomingConnections(int max_incoming_connections) { + // The maximum can only be configured once. + GPR_ASSERT(max_incoming_connections < INT_MAX); + GPR_ASSERT(max_incoming_connections_.exchange(max_incoming_connections, + std::memory_order_release) == + INT_MAX); +} + +// Returns true if the incoming connection is allowed to be accepted on the +// server. +bool ConnectionQuota::AllowIncomingConnection(MemoryQuotaRefPtr mem_quota, + absl::string_view /*peer*/) { + if (mem_quota->IsMemoryPressureHigh()) { + return false; + } + + if (max_incoming_connections_.load(std::memory_order_relaxed) == INT_MAX) { + return true; + } + + int curr_active_connections = + active_incoming_connections_.load(std::memory_order_acquire); + do { + if (curr_active_connections >= + max_incoming_connections_.load(std::memory_order_relaxed)) { + return false; + } + } while (!active_incoming_connections_.compare_exchange_weak( + curr_active_connections, curr_active_connections + 1, + std::memory_order_acq_rel, std::memory_order_relaxed)); + return true; +} + +// Mark connections as closed. +void ConnectionQuota::ReleaseConnections(int num_connections) { + if (max_incoming_connections_.load(std::memory_order_relaxed) == INT_MAX) { + return; + } + GPR_ASSERT(active_incoming_connections_.fetch_sub( + num_connections, std::memory_order_acq_rel) >= + num_connections); +} + +} // namespace grpc_core diff --git a/src/core/lib/resource_quota/connection_quota.h b/src/core/lib/resource_quota/connection_quota.h new file mode 100644 index 00000000000..32815f57600 --- /dev/null +++ b/src/core/lib/resource_quota/connection_quota.h @@ -0,0 +1,61 @@ +// Copyright 2024 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GRPC_SRC_CORE_LIB_RESOURCE_QUOTA_CONNECTION_QUOTA_H +#define GRPC_SRC_CORE_LIB_RESOURCE_QUOTA_CONNECTION_QUOTA_H + +#include + +#include +#include + +#include "absl/base/thread_annotations.h" + +#include "src/core/lib/gprpp/ref_counted.h" +#include "src/core/lib/gprpp/ref_counted_ptr.h" +#include "src/core/lib/gprpp/sync.h" +#include "src/core/lib/resource_quota/memory_quota.h" + +namespace grpc_core { + +// Tracks the amount of threads in a resource quota. +class ConnectionQuota : public RefCounted { + public: + ConnectionQuota(); + ~ConnectionQuota() override = default; + + ConnectionQuota(const ConnectionQuota&) = delete; + ConnectionQuota& operator=(const ConnectionQuota&) = delete; + + // Set the maximum number of allowed incoming connections on the server. + void SetMaxIncomingConnections(int max_incoming_connections); + + // Returns true if the incoming connection is allowed to be accepted on the + // server. + bool AllowIncomingConnection(MemoryQuotaRefPtr mem_quota, + absl::string_view peer); + + // Mark connections as closed. + void ReleaseConnections(int num_connections); + + private: + std::atomic active_incoming_connections_{0}; + std::atomic max_incoming_connections_{std::numeric_limits::max()}; +}; + +using ConnectionQuotaRefPtr = RefCountedPtr; + +} // namespace grpc_core + +#endif // GRPC_SRC_CORE_LIB_RESOURCE_QUOTA_CONNECTION_QUOTA_H diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 857b62a8259..f3f4a909fd5 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -658,6 +658,7 @@ CORE_SOURCE_FILES = [ 'src/core/lib/promise/trace.cc', 'src/core/lib/resource_quota/api.cc', 'src/core/lib/resource_quota/arena.cc', + 'src/core/lib/resource_quota/connection_quota.cc', 'src/core/lib/resource_quota/memory_quota.cc', 'src/core/lib/resource_quota/periodic_update.cc', 'src/core/lib/resource_quota/resource_quota.cc', diff --git a/test/cpp/end2end/resource_quota_end2end_stress_test.cc b/test/cpp/end2end/resource_quota_end2end_stress_test.cc index 83af5a2fe27..4a8f28deaeb 100644 --- a/test/cpp/end2end/resource_quota_end2end_stress_test.cc +++ b/test/cpp/end2end/resource_quota_end2end_stress_test.cc @@ -53,7 +53,6 @@ class EchoClientUnaryReactor : public grpc::ClientUnaryReactor { EchoClientUnaryReactor(ClientContext* ctx, EchoTestService::Stub* stub, const std::string payload, Status* status) : ctx_(ctx), payload_(payload), status_(status) { - ctx_->set_wait_for_ready(true); request_.set_message(payload); stub->async()->Echo(ctx_, &request_, &response_, this); StartCall(); @@ -120,6 +119,7 @@ class End2EndResourceQuotaUnaryTest : public ::testing::Test { Status status; auto stub = EchoTestService::NewStub( CreateChannel(server_address_, grpc::InsecureChannelCredentials())); + ctx.set_wait_for_ready(true); EchoClientUnaryReactor reactor(&ctx, stub.get(), payload_, &status); reactor.Await(); } @@ -145,6 +145,78 @@ class End2EndResourceQuotaUnaryTest : public ::testing::Test { TEST_F(End2EndResourceQuotaUnaryTest, MultipleUnaryRPCTest) { MakeGrpcCalls(); } +class End2EndConnectionQuotaTest : public ::testing::TestWithParam { + protected: + End2EndConnectionQuotaTest() { + port_ = grpc_pick_unused_port_or_die(); + server_address_ = absl::StrCat("[::]:", port_); + payload_ = std::string(kPayloadSizeBytes, 'a'); + ServerBuilder builder; + builder.AddListeningPort(server_address_, InsecureServerCredentials()); + builder.AddChannelArgument(GRPC_ARG_MAX_ALLOWED_INCOMING_CONNECTIONS, + GetParam()); + builder.AddChannelArgument( + GRPC_ARG_HTTP2_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS, 10000); + builder.RegisterService(&grpc_service_); + server_ = builder.BuildAndStart(); + } + + ~End2EndConnectionQuotaTest() override { server_->Shutdown(); } + + std::unique_ptr CreateGrpcChannelStub() { + grpc::ChannelArguments args; + args.SetInt(GRPC_ARG_USE_LOCAL_SUBCHANNEL_POOL, 1); + args.SetInt(GRPC_ARG_ENABLE_RETRIES, 0); + args.SetInt(GRPC_ARG_KEEPALIVE_TIME_MS, 20000); + args.SetInt(GRPC_ARG_KEEPALIVE_TIMEOUT_MS, 10000); + args.SetInt(GRPC_ARG_HTTP2_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS, 15000); + args.SetInt(GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS, 1); + + return EchoTestService::NewStub( + CreateCustomChannel(absl::StrCat("ipv6:[::1]:", port_), + grpc::InsecureChannelCredentials(), args)); + } + + void TestExceedingConnectionQuota() { + const int kNumConnections = 2 * GetParam(); + std::vector> stubs; + stubs.reserve(kNumConnections); + for (int i = 0; i < kNumConnections; i++) { + stubs.push_back(CreateGrpcChannelStub()); + } + for (int i = 0; i < kNumConnections; ++i) { + ClientContext ctx; + Status status; + ctx.set_wait_for_ready(false); + EchoClientUnaryReactor reactor(&ctx, stubs[i].get(), payload_, &status); + reactor.Await(); + // The first half RPCs should succeed. + if (i < kNumConnections / 2) { + EXPECT_TRUE(status.ok()); + + } else { + // The second half should fail because they would attempt to create a + // new connection and fail since it would exceed the connection quota + // limit set at the server. + EXPECT_FALSE(status.ok()); + } + } + } + + int port_; + std::unique_ptr server_; + string server_address_; + GrpcCallbackServiceImpl grpc_service_; + std::string payload_; +}; + +TEST_P(End2EndConnectionQuotaTest, ConnectionQuotaTest) { + TestExceedingConnectionQuota(); +} + +INSTANTIATE_TEST_SUITE_P(ConnectionQuotaParamTest, End2EndConnectionQuotaTest, + ::testing::ValuesIn({10, 100})); + } // namespace testing } // namespace grpc diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 1fbf624c5d3..39f3d505ed3 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -2613,6 +2613,8 @@ src/core/lib/resource_quota/api.cc \ src/core/lib/resource_quota/api.h \ src/core/lib/resource_quota/arena.cc \ src/core/lib/resource_quota/arena.h \ +src/core/lib/resource_quota/connection_quota.cc \ +src/core/lib/resource_quota/connection_quota.h \ src/core/lib/resource_quota/memory_quota.cc \ src/core/lib/resource_quota/memory_quota.h \ src/core/lib/resource_quota/periodic_update.cc \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 9f4eb9b2777..7ef0084dfc4 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -2388,6 +2388,8 @@ src/core/lib/resource_quota/api.cc \ src/core/lib/resource_quota/api.h \ src/core/lib/resource_quota/arena.cc \ src/core/lib/resource_quota/arena.h \ +src/core/lib/resource_quota/connection_quota.cc \ +src/core/lib/resource_quota/connection_quota.h \ src/core/lib/resource_quota/memory_quota.cc \ src/core/lib/resource_quota/memory_quota.h \ src/core/lib/resource_quota/periodic_update.cc \ From 7682a318d85d2fa96e04d05bf10464734f82f26a Mon Sep 17 00:00:00 2001 From: Vignesh Babu Date: Mon, 11 Mar 2024 15:35:06 -0700 Subject: [PATCH 27/52] Creating a chaotic good endpoint extension to allow configuring chaotic good endpoints. The extension first contains methods to enable logging of stats. PiperOrigin-RevId: 614816675 --- Package.swift | 1 + build_autogenerated.yaml | 3 + gRPC-C++.podspec | 2 + gRPC-Core.podspec | 2 + grpc.gemspec | 1 + package.xml | 1 + src/core/BUILD | 5 + .../client/chaotic_good_connector.cc | 17 ++ .../server/chaotic_good_server.cc | 36 +-- src/core/lib/debug/stats_data.cc | 228 ++++++++++++++++-- src/core/lib/debug/stats_data.h | 108 +++++++++ src/core/lib/debug/stats_data.yaml | 57 +++++ .../extensions/chaotic_good_extension.h | 45 ++++ src/core/lib/event_engine/posix.h | 8 + tools/doxygen/Doxyfile.c++.internal | 1 + tools/doxygen/Doxyfile.core.internal | 1 + 16 files changed, 477 insertions(+), 39 deletions(-) create mode 100644 src/core/lib/event_engine/extensions/chaotic_good_extension.h diff --git a/Package.swift b/Package.swift index 9737494d3bc..5e15d8e55af 100644 --- a/Package.swift +++ b/Package.swift @@ -1202,6 +1202,7 @@ let package = Package( "src/core/lib/event_engine/default_event_engine_factory.h", "src/core/lib/event_engine/event_engine.cc", "src/core/lib/event_engine/extensions/can_track_errors.h", + "src/core/lib/event_engine/extensions/chaotic_good_extension.h", "src/core/lib/event_engine/extensions/supports_fd.h", "src/core/lib/event_engine/forkable.cc", "src/core/lib/event_engine/forkable.h", diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index e9478a2a4b8..a3f46868f21 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -845,6 +845,7 @@ libs: - src/core/lib/event_engine/default_event_engine.h - src/core/lib/event_engine/default_event_engine_factory.h - src/core/lib/event_engine/extensions/can_track_errors.h + - src/core/lib/event_engine/extensions/chaotic_good_extension.h - src/core/lib/event_engine/extensions/supports_fd.h - src/core/lib/event_engine/forkable.h - src/core/lib/event_engine/grpc_polled_fd.h @@ -2353,6 +2354,7 @@ libs: - src/core/lib/event_engine/default_event_engine.h - src/core/lib/event_engine/default_event_engine_factory.h - src/core/lib/event_engine/extensions/can_track_errors.h + - src/core/lib/event_engine/extensions/chaotic_good_extension.h - src/core/lib/event_engine/extensions/supports_fd.h - src/core/lib/event_engine/forkable.h - src/core/lib/event_engine/grpc_polled_fd.h @@ -4439,6 +4441,7 @@ libs: - src/core/lib/event_engine/default_event_engine.h - src/core/lib/event_engine/default_event_engine_factory.h - src/core/lib/event_engine/extensions/can_track_errors.h + - src/core/lib/event_engine/extensions/chaotic_good_extension.h - src/core/lib/event_engine/extensions/supports_fd.h - src/core/lib/event_engine/forkable.h - src/core/lib/event_engine/grpc_polled_fd.h diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index bad4a958ef8..898b33d3655 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -928,6 +928,7 @@ Pod::Spec.new do |s| 'src/core/lib/event_engine/default_event_engine.h', 'src/core/lib/event_engine/default_event_engine_factory.h', 'src/core/lib/event_engine/extensions/can_track_errors.h', + 'src/core/lib/event_engine/extensions/chaotic_good_extension.h', 'src/core/lib/event_engine/extensions/supports_fd.h', 'src/core/lib/event_engine/forkable.h', 'src/core/lib/event_engine/grpc_polled_fd.h', @@ -2192,6 +2193,7 @@ Pod::Spec.new do |s| 'src/core/lib/event_engine/default_event_engine.h', 'src/core/lib/event_engine/default_event_engine_factory.h', 'src/core/lib/event_engine/extensions/can_track_errors.h', + 'src/core/lib/event_engine/extensions/chaotic_good_extension.h', 'src/core/lib/event_engine/extensions/supports_fd.h', 'src/core/lib/event_engine/forkable.h', 'src/core/lib/event_engine/grpc_polled_fd.h', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index ca1624e7ac2..1c01a1e54c0 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -1316,6 +1316,7 @@ Pod::Spec.new do |s| 'src/core/lib/event_engine/default_event_engine_factory.h', 'src/core/lib/event_engine/event_engine.cc', 'src/core/lib/event_engine/extensions/can_track_errors.h', + 'src/core/lib/event_engine/extensions/chaotic_good_extension.h', 'src/core/lib/event_engine/extensions/supports_fd.h', 'src/core/lib/event_engine/forkable.cc', 'src/core/lib/event_engine/forkable.h', @@ -2974,6 +2975,7 @@ Pod::Spec.new do |s| 'src/core/lib/event_engine/default_event_engine.h', 'src/core/lib/event_engine/default_event_engine_factory.h', 'src/core/lib/event_engine/extensions/can_track_errors.h', + 'src/core/lib/event_engine/extensions/chaotic_good_extension.h', 'src/core/lib/event_engine/extensions/supports_fd.h', 'src/core/lib/event_engine/forkable.h', 'src/core/lib/event_engine/grpc_polled_fd.h', diff --git a/grpc.gemspec b/grpc.gemspec index 12d4db8b99b..d5eca0ba3cb 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -1208,6 +1208,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/event_engine/default_event_engine_factory.h ) s.files += %w( src/core/lib/event_engine/event_engine.cc ) s.files += %w( src/core/lib/event_engine/extensions/can_track_errors.h ) + s.files += %w( src/core/lib/event_engine/extensions/chaotic_good_extension.h ) s.files += %w( src/core/lib/event_engine/extensions/supports_fd.h ) s.files += %w( src/core/lib/event_engine/forkable.cc ) s.files += %w( src/core/lib/event_engine/forkable.h ) diff --git a/package.xml b/package.xml index 1ba949ae253..901a73b61b0 100644 --- a/package.xml +++ b/package.xml @@ -1190,6 +1190,7 @@ + diff --git a/src/core/BUILD b/src/core/BUILD index 46708db643f..c591f75aee0 100644 --- a/src/core/BUILD +++ b/src/core/BUILD @@ -59,6 +59,7 @@ grpc_cc_library( name = "event_engine_extensions", hdrs = [ "lib/event_engine/extensions/can_track_errors.h", + "lib/event_engine/extensions/chaotic_good_extension.h", "lib/event_engine/extensions/supports_fd.h", ], external_deps = [ @@ -7482,6 +7483,8 @@ grpc_cc_library( "error", "error_utils", "event_engine_common", + "event_engine_extensions", + "event_engine_query_extensions", "event_engine_tcp_socket_utils", "event_engine_wakeup_scheduler", "grpc_promise_endpoint", @@ -7543,6 +7546,8 @@ grpc_cc_library( "default_event_engine", "error", "error_utils", + "event_engine_extensions", + "event_engine_query_extensions", "event_engine_tcp_socket_utils", "event_engine_wakeup_scheduler", "grpc_promise_endpoint", diff --git a/src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc b/src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc index adc2aad7dd3..4fc5cc9dc56 100644 --- a/src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc +++ b/src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc @@ -35,6 +35,8 @@ #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/event_engine/channel_args_endpoint_config.h" #include "src/core/lib/event_engine/default_event_engine.h" +#include "src/core/lib/event_engine/extensions/chaotic_good_extension.h" +#include "src/core/lib/event_engine/query_extensions.h" #include "src/core/lib/event_engine/tcp_socket_utils.h" #include "src/core/lib/gprpp/debug_location.h" #include "src/core/lib/gprpp/no_destruct.h" @@ -130,6 +132,14 @@ auto ChaoticGoodConnector::WaitForDataEndpointSetup( GRPC_ERROR_CREATE("connect endpoint failed")); return; } + auto* chaotic_good_ext = + grpc_event_engine::experimental::QueryExtension< + grpc_event_engine::experimental::ChaoticGoodExtension>( + endpoint.value().get()); + if (chaotic_good_ext != nullptr) { + chaotic_good_ext->EnableStatsCollection( + /*is_control_channel=*/false); + } self->data_endpoint_ = PromiseEndpoint(std::move(endpoint.value()), SliceBuffer()); self->data_endpoint_ready_.Set(); @@ -241,6 +251,13 @@ void ChaoticGoodConnector::Connect(const Args& args, Result* result, return; } auto* p = self.release(); + auto* chaotic_good_ext = + grpc_event_engine::experimental::QueryExtension< + grpc_event_engine::experimental::ChaoticGoodExtension>( + endpoint.value().get()); + if (chaotic_good_ext != nullptr) { + chaotic_good_ext->EnableStatsCollection(/*is_control_channel=*/true); + } p->handshake_mgr_->DoHandshake( grpc_event_engine_endpoint_create(std::move(endpoint.value())), p->args_.channel_args, p->args_.deadline, nullptr /* acceptor */, diff --git a/src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc b/src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc index c7815f8b9a2..ec0bcac7358 100644 --- a/src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc +++ b/src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc @@ -39,6 +39,8 @@ #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/event_engine/channel_args_endpoint_config.h" #include "src/core/lib/event_engine/default_event_engine.h" +#include "src/core/lib/event_engine/extensions/chaotic_good_extension.h" +#include "src/core/lib/event_engine/query_extensions.h" #include "src/core/lib/event_engine/resolved_address_internal.h" #include "src/core/lib/event_engine/tcp_socket_utils.h" #include "src/core/lib/gprpp/orphanable.h" @@ -397,22 +399,28 @@ void ChaoticGoodServerListener::ActiveConnection::HandshakingState:: } GPR_ASSERT(grpc_event_engine::experimental::grpc_is_event_engine_endpoint( args->endpoint)); - self->connection_->endpoint_ = PromiseEndpoint( + auto ee_endpoint = grpc_event_engine::experimental::grpc_take_wrapped_event_engine_endpoint( - args->endpoint), - SliceBuffer()); + args->endpoint); + auto* chaotic_good_ext = grpc_event_engine::experimental::QueryExtension< + grpc_event_engine::experimental::ChaoticGoodExtension>(ee_endpoint.get()); + self->connection_->endpoint_ = + PromiseEndpoint(std::move(ee_endpoint), SliceBuffer()); auto activity = MakeActivity( - [self]() { - return TrySeq(Race(EndpointReadSettingsFrame(self), - TrySeq(Sleep(Timestamp::Now() + kConnectionDeadline), - []() -> absl::StatusOr { - return absl::DeadlineExceededError( - "Waiting for initial settings frame"); - })), - [self](bool is_control_endpoint) { - return EndpointWriteSettingsFrame(self, - is_control_endpoint); - }); + [self, chaotic_good_ext]() { + return TrySeq( + Race(EndpointReadSettingsFrame(self), + TrySeq(Sleep(Timestamp::Now() + kConnectionDeadline), + []() -> absl::StatusOr { + return absl::DeadlineExceededError( + "Waiting for initial settings frame"); + })), + [self, chaotic_good_ext](bool is_control_endpoint) { + if (chaotic_good_ext != nullptr) { + chaotic_good_ext->EnableStatsCollection(is_control_endpoint); + } + return EndpointWriteSettingsFrame(self, is_control_endpoint); + }); }, EventEngineWakeupScheduler(self->connection_->listener_->event_engine_), [self](absl::Status status) { diff --git a/src/core/lib/debug/stats_data.cc b/src/core/lib/debug/stats_data.cc index bed24de478f..4a08fe5ae5e 100644 --- a/src/core/lib/debug/stats_data.cc +++ b/src/core/lib/debug/stats_data.cc @@ -53,6 +53,19 @@ Histogram_65536_26 operator-(const Histogram_65536_26& left, } return result; } +void HistogramCollector_100_20::Collect(Histogram_100_20* result) const { + for (int i = 0; i < 20; i++) { + result->buckets_[i] += buckets_[i].load(std::memory_order_relaxed); + } +} +Histogram_100_20 operator-(const Histogram_100_20& left, + const Histogram_100_20& right) { + Histogram_100_20 result; + for (int i = 0; i < 20; i++) { + result.buckets_[i] = left.buckets_[i] - right.buckets_[i]; + } + return result; +} void HistogramCollector_16777216_20::Collect( Histogram_16777216_20* result) const { for (int i = 0; i < 20; i++) { @@ -185,6 +198,20 @@ const absl::string_view "work_serializer_work_time_ms", "work_serializer_work_time_per_item_ms", "work_serializer_items_per_run", + "chaotic_good_sendmsgs_per_write_control", + "chaotic_good_recvmsgs_per_read_control", + "chaotic_good_sendmsgs_per_write_data", + "chaotic_good_recvmsgs_per_read_data", + "chaotic_good_thread_hops_per_write_control", + "chaotic_good_thread_hops_per_read_control", + "chaotic_good_thread_hops_per_write_data", + "chaotic_good_thread_hops_per_read_data", + "chaotic_good_tcp_read_size_data", + "chaotic_good_tcp_read_size_control", + "chaotic_good_tcp_read_offer_data", + "chaotic_good_tcp_read_offer_control", + "chaotic_good_tcp_write_size_data", + "chaotic_good_tcp_write_size_control", }; const absl::string_view GlobalStats::histogram_doc[static_cast( Histogram::COUNT)] = { @@ -203,6 +230,20 @@ const absl::string_view GlobalStats::histogram_doc[static_cast( "work", "How long do individual items take to process in work serializers", "How many callbacks are executed when a work serializer runs", + "Number of sendmsgs per control channel endpoint write", + "Number of recvmsgs per control channel endpoint read", + "Number of sendmsgs per data channel endpoint write", + "Number of recvmsgs per data channel endpoint read", + "Number of thread hops per control channel endpoint write", + "Number of thread hops per control channel endpoint read", + "Number of thread hops per data channel endpoint write", + "Number of thread hops per data channel endpoint read", + "Number of bytes received by each syscall_read in the data channel", + "Number of bytes received by each syscall_read in the control channel", + "Number of bytes offered to each syscall_read in the data channel", + "Number of bytes offered to each syscall_read in the control channel", + "Number of bytes offered to each syscall_write in the data channel", + "Number of bytes offered to each syscall_write in the control channel", }; namespace { const int kStatsTable0[21] = {0, 1, 2, 4, 8, 15, 27, @@ -218,21 +259,25 @@ const int kStatsTable2[27] = {0, 1, 2, 4, 7, 11, 17, const uint8_t kStatsTable3[29] = {3, 3, 4, 5, 6, 6, 7, 8, 9, 10, 11, 11, 12, 13, 14, 15, 16, 16, 17, 18, 19, 20, 21, 21, 22, 23, 24, 25, 26}; -const int kStatsTable4[21] = { +const int kStatsTable4[21] = {0, 1, 2, 3, 4, 5, 7, 9, 11, 14, 17, + 21, 25, 30, 36, 43, 51, 61, 72, 85, 100}; +const uint8_t kStatsTable5[16] = {6, 6, 7, 8, 9, 9, 10, 11, + 12, 13, 14, 15, 16, 17, 18, 19}; +const int kStatsTable6[21] = { 0, 1, 3, 8, 19, 45, 106, 250, 588, 1383, 3252, 7646, 17976, 42262, 99359, 233593, 549177, 1291113, 3035402, 7136218, 16777216}; -const uint8_t kStatsTable5[23] = {2, 3, 3, 4, 5, 6, 7, 8, +const uint8_t kStatsTable7[23] = {2, 3, 3, 4, 5, 6, 7, 8, 8, 9, 10, 11, 12, 12, 13, 14, 15, 16, 16, 17, 18, 19, 20}; -const int kStatsTable6[11] = {0, 1, 2, 4, 7, 11, 17, 26, 38, 56, 80}; -const uint8_t kStatsTable7[9] = {3, 3, 4, 5, 6, 6, 7, 8, 9}; -const int kStatsTable8[21] = {0, 1, 2, 4, 7, 12, 19, - 30, 47, 74, 116, 182, 285, 445, - 695, 1084, 1691, 2637, 4113, 6414, 10000}; -const uint8_t kStatsTable9[23] = {3, 3, 4, 5, 5, 6, 7, 8, - 9, 9, 10, 11, 12, 12, 13, 14, - 15, 15, 16, 17, 18, 18, 19}; +const int kStatsTable8[11] = {0, 1, 2, 4, 7, 11, 17, 26, 38, 56, 80}; +const uint8_t kStatsTable9[9] = {3, 3, 4, 5, 6, 6, 7, 8, 9}; +const int kStatsTable10[21] = {0, 1, 2, 4, 7, 12, 19, + 30, 47, 74, 116, 182, 285, 445, + 695, 1084, 1691, 2637, 4113, 6414, 10000}; +const uint8_t kStatsTable11[23] = {3, 3, 4, 5, 5, 6, 7, 8, + 9, 9, 10, 11, 12, 12, 13, 14, + 15, 15, 16, 17, 18, 18, 19}; } // namespace int Histogram_100000_20::BucketFor(int value) { if (value < 3) { @@ -272,6 +317,29 @@ int Histogram_65536_26::BucketFor(int value) { } } } +int Histogram_100_20::BucketFor(int value) { + if (value < 6) { + if (value < 0) { + return 0; + } else { + return value; + } + } else { + if (value < 81) { + DblUint val; + val.dbl = value; + const int bucket = + kStatsTable5[((val.uint - 4618441417868443648ull) >> 50)]; + return bucket - (value < kStatsTable4[bucket]); + } else { + if (value < 85) { + return 18; + } else { + return 19; + } + } + } +} int Histogram_16777216_20::BucketFor(int value) { if (value < 2) { if (value < 0) { @@ -284,8 +352,8 @@ int Histogram_16777216_20::BucketFor(int value) { DblUint val; val.dbl = value; const int bucket = - kStatsTable5[((val.uint - 4611686018427387904ull) >> 52)]; - return bucket - (value < kStatsTable4[bucket]); + kStatsTable7[((val.uint - 4611686018427387904ull) >> 52)]; + return bucket - (value < kStatsTable6[bucket]); } else { return 19; } @@ -303,8 +371,8 @@ int Histogram_80_10::BucketFor(int value) { DblUint val; val.dbl = value; const int bucket = - kStatsTable7[((val.uint - 4613937818241073152ull) >> 51)]; - return bucket - (value < kStatsTable6[bucket]); + kStatsTable9[((val.uint - 4613937818241073152ull) >> 51)]; + return bucket - (value < kStatsTable8[bucket]); } else { if (value < 56) { return 8; @@ -326,8 +394,8 @@ int Histogram_10000_20::BucketFor(int value) { DblUint val; val.dbl = value; const int bucket = - kStatsTable9[((val.uint - 4613937818241073152ull) >> 51)]; - return bucket - (value < kStatsTable8[bucket]); + kStatsTable11[((val.uint - 4613937818241073152ull) >> 51)]; + return bucket - (value < kStatsTable10[bucket]); } else { if (value < 6414) { return 18; @@ -378,31 +446,31 @@ HistogramView GlobalStats::histogram(Histogram which) const { return HistogramView{&Histogram_65536_26::BucketFor, kStatsTable2, 26, call_initial_size.buckets()}; case Histogram::kTcpWriteSize: - return HistogramView{&Histogram_16777216_20::BucketFor, kStatsTable4, 20, + return HistogramView{&Histogram_16777216_20::BucketFor, kStatsTable6, 20, tcp_write_size.buckets()}; case Histogram::kTcpWriteIovSize: - return HistogramView{&Histogram_80_10::BucketFor, kStatsTable6, 10, + return HistogramView{&Histogram_80_10::BucketFor, kStatsTable8, 10, tcp_write_iov_size.buckets()}; case Histogram::kTcpReadSize: - return HistogramView{&Histogram_16777216_20::BucketFor, kStatsTable4, 20, + return HistogramView{&Histogram_16777216_20::BucketFor, kStatsTable6, 20, tcp_read_size.buckets()}; case Histogram::kTcpReadOffer: - return HistogramView{&Histogram_16777216_20::BucketFor, kStatsTable4, 20, + return HistogramView{&Histogram_16777216_20::BucketFor, kStatsTable6, 20, tcp_read_offer.buckets()}; case Histogram::kTcpReadOfferIovSize: - return HistogramView{&Histogram_80_10::BucketFor, kStatsTable6, 10, + return HistogramView{&Histogram_80_10::BucketFor, kStatsTable8, 10, tcp_read_offer_iov_size.buckets()}; case Histogram::kHttp2SendMessageSize: - return HistogramView{&Histogram_16777216_20::BucketFor, kStatsTable4, 20, + return HistogramView{&Histogram_16777216_20::BucketFor, kStatsTable6, 20, http2_send_message_size.buckets()}; case Histogram::kHttp2MetadataSize: return HistogramView{&Histogram_65536_26::BucketFor, kStatsTable2, 26, http2_metadata_size.buckets()}; case Histogram::kWrrSubchannelListSize: - return HistogramView{&Histogram_10000_20::BucketFor, kStatsTable8, 20, + return HistogramView{&Histogram_10000_20::BucketFor, kStatsTable10, 20, wrr_subchannel_list_size.buckets()}; case Histogram::kWrrSubchannelReadySize: - return HistogramView{&Histogram_10000_20::BucketFor, kStatsTable8, 20, + return HistogramView{&Histogram_10000_20::BucketFor, kStatsTable10, 20, wrr_subchannel_ready_size.buckets()}; case Histogram::kWorkSerializerRunTimeMs: return HistogramView{&Histogram_100000_20::BucketFor, kStatsTable0, 20, @@ -414,8 +482,51 @@ HistogramView GlobalStats::histogram(Histogram which) const { return HistogramView{&Histogram_100000_20::BucketFor, kStatsTable0, 20, work_serializer_work_time_per_item_ms.buckets()}; case Histogram::kWorkSerializerItemsPerRun: - return HistogramView{&Histogram_10000_20::BucketFor, kStatsTable8, 20, + return HistogramView{&Histogram_10000_20::BucketFor, kStatsTable10, 20, work_serializer_items_per_run.buckets()}; + case Histogram::kChaoticGoodSendmsgsPerWriteControl: + return HistogramView{&Histogram_100_20::BucketFor, kStatsTable4, 20, + chaotic_good_sendmsgs_per_write_control.buckets()}; + case Histogram::kChaoticGoodRecvmsgsPerReadControl: + return HistogramView{&Histogram_100_20::BucketFor, kStatsTable4, 20, + chaotic_good_recvmsgs_per_read_control.buckets()}; + case Histogram::kChaoticGoodSendmsgsPerWriteData: + return HistogramView{&Histogram_100_20::BucketFor, kStatsTable4, 20, + chaotic_good_sendmsgs_per_write_data.buckets()}; + case Histogram::kChaoticGoodRecvmsgsPerReadData: + return HistogramView{&Histogram_100_20::BucketFor, kStatsTable4, 20, + chaotic_good_recvmsgs_per_read_data.buckets()}; + case Histogram::kChaoticGoodThreadHopsPerWriteControl: + return HistogramView{ + &Histogram_100_20::BucketFor, kStatsTable4, 20, + chaotic_good_thread_hops_per_write_control.buckets()}; + case Histogram::kChaoticGoodThreadHopsPerReadControl: + return HistogramView{&Histogram_100_20::BucketFor, kStatsTable4, 20, + chaotic_good_thread_hops_per_read_control.buckets()}; + case Histogram::kChaoticGoodThreadHopsPerWriteData: + return HistogramView{&Histogram_100_20::BucketFor, kStatsTable4, 20, + chaotic_good_thread_hops_per_write_data.buckets()}; + case Histogram::kChaoticGoodThreadHopsPerReadData: + return HistogramView{&Histogram_100_20::BucketFor, kStatsTable4, 20, + chaotic_good_thread_hops_per_read_data.buckets()}; + case Histogram::kChaoticGoodTcpReadSizeData: + return HistogramView{&Histogram_16777216_20::BucketFor, kStatsTable6, 20, + chaotic_good_tcp_read_size_data.buckets()}; + case Histogram::kChaoticGoodTcpReadSizeControl: + return HistogramView{&Histogram_16777216_20::BucketFor, kStatsTable6, 20, + chaotic_good_tcp_read_size_control.buckets()}; + case Histogram::kChaoticGoodTcpReadOfferData: + return HistogramView{&Histogram_16777216_20::BucketFor, kStatsTable6, 20, + chaotic_good_tcp_read_offer_data.buckets()}; + case Histogram::kChaoticGoodTcpReadOfferControl: + return HistogramView{&Histogram_16777216_20::BucketFor, kStatsTable6, 20, + chaotic_good_tcp_read_offer_control.buckets()}; + case Histogram::kChaoticGoodTcpWriteSizeData: + return HistogramView{&Histogram_16777216_20::BucketFor, kStatsTable6, 20, + chaotic_good_tcp_write_size_data.buckets()}; + case Histogram::kChaoticGoodTcpWriteSizeControl: + return HistogramView{&Histogram_16777216_20::BucketFor, kStatsTable6, 20, + chaotic_good_tcp_write_size_control.buckets()}; } } std::unique_ptr GlobalStatsCollector::Collect() const { @@ -497,6 +608,34 @@ std::unique_ptr GlobalStatsCollector::Collect() const { &result->work_serializer_work_time_per_item_ms); data.work_serializer_items_per_run.Collect( &result->work_serializer_items_per_run); + data.chaotic_good_sendmsgs_per_write_control.Collect( + &result->chaotic_good_sendmsgs_per_write_control); + data.chaotic_good_recvmsgs_per_read_control.Collect( + &result->chaotic_good_recvmsgs_per_read_control); + data.chaotic_good_sendmsgs_per_write_data.Collect( + &result->chaotic_good_sendmsgs_per_write_data); + data.chaotic_good_recvmsgs_per_read_data.Collect( + &result->chaotic_good_recvmsgs_per_read_data); + data.chaotic_good_thread_hops_per_write_control.Collect( + &result->chaotic_good_thread_hops_per_write_control); + data.chaotic_good_thread_hops_per_read_control.Collect( + &result->chaotic_good_thread_hops_per_read_control); + data.chaotic_good_thread_hops_per_write_data.Collect( + &result->chaotic_good_thread_hops_per_write_data); + data.chaotic_good_thread_hops_per_read_data.Collect( + &result->chaotic_good_thread_hops_per_read_data); + data.chaotic_good_tcp_read_size_data.Collect( + &result->chaotic_good_tcp_read_size_data); + data.chaotic_good_tcp_read_size_control.Collect( + &result->chaotic_good_tcp_read_size_control); + data.chaotic_good_tcp_read_offer_data.Collect( + &result->chaotic_good_tcp_read_offer_data); + data.chaotic_good_tcp_read_offer_control.Collect( + &result->chaotic_good_tcp_read_offer_control); + data.chaotic_good_tcp_write_size_data.Collect( + &result->chaotic_good_tcp_write_size_data); + data.chaotic_good_tcp_write_size_control.Collect( + &result->chaotic_good_tcp_write_size_control); } return result; } @@ -569,6 +708,45 @@ std::unique_ptr GlobalStats::Diff(const GlobalStats& other) const { other.work_serializer_work_time_per_item_ms; result->work_serializer_items_per_run = work_serializer_items_per_run - other.work_serializer_items_per_run; + result->chaotic_good_sendmsgs_per_write_control = + chaotic_good_sendmsgs_per_write_control - + other.chaotic_good_sendmsgs_per_write_control; + result->chaotic_good_recvmsgs_per_read_control = + chaotic_good_recvmsgs_per_read_control - + other.chaotic_good_recvmsgs_per_read_control; + result->chaotic_good_sendmsgs_per_write_data = + chaotic_good_sendmsgs_per_write_data - + other.chaotic_good_sendmsgs_per_write_data; + result->chaotic_good_recvmsgs_per_read_data = + chaotic_good_recvmsgs_per_read_data - + other.chaotic_good_recvmsgs_per_read_data; + result->chaotic_good_thread_hops_per_write_control = + chaotic_good_thread_hops_per_write_control - + other.chaotic_good_thread_hops_per_write_control; + result->chaotic_good_thread_hops_per_read_control = + chaotic_good_thread_hops_per_read_control - + other.chaotic_good_thread_hops_per_read_control; + result->chaotic_good_thread_hops_per_write_data = + chaotic_good_thread_hops_per_write_data - + other.chaotic_good_thread_hops_per_write_data; + result->chaotic_good_thread_hops_per_read_data = + chaotic_good_thread_hops_per_read_data - + other.chaotic_good_thread_hops_per_read_data; + result->chaotic_good_tcp_read_size_data = + chaotic_good_tcp_read_size_data - other.chaotic_good_tcp_read_size_data; + result->chaotic_good_tcp_read_size_control = + chaotic_good_tcp_read_size_control - + other.chaotic_good_tcp_read_size_control; + result->chaotic_good_tcp_read_offer_data = + chaotic_good_tcp_read_offer_data - other.chaotic_good_tcp_read_offer_data; + result->chaotic_good_tcp_read_offer_control = + chaotic_good_tcp_read_offer_control - + other.chaotic_good_tcp_read_offer_control; + result->chaotic_good_tcp_write_size_data = + chaotic_good_tcp_write_size_data - other.chaotic_good_tcp_write_size_data; + result->chaotic_good_tcp_write_size_control = + chaotic_good_tcp_write_size_control - + other.chaotic_good_tcp_write_size_control; return result; } } // namespace grpc_core diff --git a/src/core/lib/debug/stats_data.h b/src/core/lib/debug/stats_data.h index fd8d3415ecd..ec4a6b8765c 100644 --- a/src/core/lib/debug/stats_data.h +++ b/src/core/lib/debug/stats_data.h @@ -76,6 +76,29 @@ class HistogramCollector_65536_26 { private: std::atomic buckets_[26]{}; }; +class HistogramCollector_100_20; +class Histogram_100_20 { + public: + static int BucketFor(int value); + const uint64_t* buckets() const { return buckets_; } + friend Histogram_100_20 operator-(const Histogram_100_20& left, + const Histogram_100_20& right); + + private: + friend class HistogramCollector_100_20; + uint64_t buckets_[20]{}; +}; +class HistogramCollector_100_20 { + public: + void Increment(int value) { + buckets_[Histogram_100_20::BucketFor(value)].fetch_add( + 1, std::memory_order_relaxed); + } + void Collect(Histogram_100_20* result) const; + + private: + std::atomic buckets_[20]{}; +}; class HistogramCollector_16777216_20; class Histogram_16777216_20 { public: @@ -196,6 +219,20 @@ struct GlobalStats { kWorkSerializerWorkTimeMs, kWorkSerializerWorkTimePerItemMs, kWorkSerializerItemsPerRun, + kChaoticGoodSendmsgsPerWriteControl, + kChaoticGoodRecvmsgsPerReadControl, + kChaoticGoodSendmsgsPerWriteData, + kChaoticGoodRecvmsgsPerReadData, + kChaoticGoodThreadHopsPerWriteControl, + kChaoticGoodThreadHopsPerReadControl, + kChaoticGoodThreadHopsPerWriteData, + kChaoticGoodThreadHopsPerReadData, + kChaoticGoodTcpReadSizeData, + kChaoticGoodTcpReadSizeControl, + kChaoticGoodTcpReadOfferData, + kChaoticGoodTcpReadOfferControl, + kChaoticGoodTcpWriteSizeData, + kChaoticGoodTcpWriteSizeControl, COUNT }; GlobalStats(); @@ -256,6 +293,20 @@ struct GlobalStats { Histogram_100000_20 work_serializer_work_time_ms; Histogram_100000_20 work_serializer_work_time_per_item_ms; Histogram_10000_20 work_serializer_items_per_run; + Histogram_100_20 chaotic_good_sendmsgs_per_write_control; + Histogram_100_20 chaotic_good_recvmsgs_per_read_control; + Histogram_100_20 chaotic_good_sendmsgs_per_write_data; + Histogram_100_20 chaotic_good_recvmsgs_per_read_data; + Histogram_100_20 chaotic_good_thread_hops_per_write_control; + Histogram_100_20 chaotic_good_thread_hops_per_read_control; + Histogram_100_20 chaotic_good_thread_hops_per_write_data; + Histogram_100_20 chaotic_good_thread_hops_per_read_data; + Histogram_16777216_20 chaotic_good_tcp_read_size_data; + Histogram_16777216_20 chaotic_good_tcp_read_size_control; + Histogram_16777216_20 chaotic_good_tcp_read_offer_data; + Histogram_16777216_20 chaotic_good_tcp_read_offer_control; + Histogram_16777216_20 chaotic_good_tcp_write_size_data; + Histogram_16777216_20 chaotic_good_tcp_write_size_control; HistogramView histogram(Histogram which) const; std::unique_ptr Diff(const GlobalStats& other) const; }; @@ -414,6 +465,49 @@ class GlobalStatsCollector { void IncrementWorkSerializerItemsPerRun(int value) { data_.this_cpu().work_serializer_items_per_run.Increment(value); } + void IncrementChaoticGoodSendmsgsPerWriteControl(int value) { + data_.this_cpu().chaotic_good_sendmsgs_per_write_control.Increment(value); + } + void IncrementChaoticGoodRecvmsgsPerReadControl(int value) { + data_.this_cpu().chaotic_good_recvmsgs_per_read_control.Increment(value); + } + void IncrementChaoticGoodSendmsgsPerWriteData(int value) { + data_.this_cpu().chaotic_good_sendmsgs_per_write_data.Increment(value); + } + void IncrementChaoticGoodRecvmsgsPerReadData(int value) { + data_.this_cpu().chaotic_good_recvmsgs_per_read_data.Increment(value); + } + void IncrementChaoticGoodThreadHopsPerWriteControl(int value) { + data_.this_cpu().chaotic_good_thread_hops_per_write_control.Increment( + value); + } + void IncrementChaoticGoodThreadHopsPerReadControl(int value) { + data_.this_cpu().chaotic_good_thread_hops_per_read_control.Increment(value); + } + void IncrementChaoticGoodThreadHopsPerWriteData(int value) { + data_.this_cpu().chaotic_good_thread_hops_per_write_data.Increment(value); + } + void IncrementChaoticGoodThreadHopsPerReadData(int value) { + data_.this_cpu().chaotic_good_thread_hops_per_read_data.Increment(value); + } + void IncrementChaoticGoodTcpReadSizeData(int value) { + data_.this_cpu().chaotic_good_tcp_read_size_data.Increment(value); + } + void IncrementChaoticGoodTcpReadSizeControl(int value) { + data_.this_cpu().chaotic_good_tcp_read_size_control.Increment(value); + } + void IncrementChaoticGoodTcpReadOfferData(int value) { + data_.this_cpu().chaotic_good_tcp_read_offer_data.Increment(value); + } + void IncrementChaoticGoodTcpReadOfferControl(int value) { + data_.this_cpu().chaotic_good_tcp_read_offer_control.Increment(value); + } + void IncrementChaoticGoodTcpWriteSizeData(int value) { + data_.this_cpu().chaotic_good_tcp_write_size_data.Increment(value); + } + void IncrementChaoticGoodTcpWriteSizeControl(int value) { + data_.this_cpu().chaotic_good_tcp_write_size_control.Increment(value); + } private: struct Data { @@ -463,6 +557,20 @@ class GlobalStatsCollector { HistogramCollector_100000_20 work_serializer_work_time_ms; HistogramCollector_100000_20 work_serializer_work_time_per_item_ms; HistogramCollector_10000_20 work_serializer_items_per_run; + HistogramCollector_100_20 chaotic_good_sendmsgs_per_write_control; + HistogramCollector_100_20 chaotic_good_recvmsgs_per_read_control; + HistogramCollector_100_20 chaotic_good_sendmsgs_per_write_data; + HistogramCollector_100_20 chaotic_good_recvmsgs_per_read_data; + HistogramCollector_100_20 chaotic_good_thread_hops_per_write_control; + HistogramCollector_100_20 chaotic_good_thread_hops_per_read_control; + HistogramCollector_100_20 chaotic_good_thread_hops_per_write_data; + HistogramCollector_100_20 chaotic_good_thread_hops_per_read_data; + HistogramCollector_16777216_20 chaotic_good_tcp_read_size_data; + HistogramCollector_16777216_20 chaotic_good_tcp_read_size_control; + HistogramCollector_16777216_20 chaotic_good_tcp_read_offer_data; + HistogramCollector_16777216_20 chaotic_good_tcp_read_offer_control; + HistogramCollector_16777216_20 chaotic_good_tcp_write_size_data; + HistogramCollector_16777216_20 chaotic_good_tcp_write_size_control; }; PerCpu data_{PerCpuOptions().SetCpusPerShard(4).SetMaxShards(32)}; }; diff --git a/src/core/lib/debug/stats_data.yaml b/src/core/lib/debug/stats_data.yaml index 8fe25f89d54..3d96ddfff6c 100644 --- a/src/core/lib/debug/stats_data.yaml +++ b/src/core/lib/debug/stats_data.yaml @@ -141,3 +141,60 @@ doc: Number of uncommon io errors - counter: msg_errqueue_error_count doc: Number of uncommon errors returned by MSG_ERRQUEUE +- histogram: chaotic_good_sendmsgs_per_write_control + doc: Number of sendmsgs per control channel endpoint write + max: 100 + buckets: 20 +- histogram: chaotic_good_recvmsgs_per_read_control + doc: Number of recvmsgs per control channel endpoint read + max: 100 + buckets: 20 +- histogram: chaotic_good_sendmsgs_per_write_data + doc: Number of sendmsgs per data channel endpoint write + max: 100 + buckets: 20 +- histogram: chaotic_good_recvmsgs_per_read_data + doc: Number of recvmsgs per data channel endpoint read + max: 100 + buckets: 20 +- histogram: chaotic_good_thread_hops_per_write_control + doc: Number of thread hops per control channel endpoint write + max: 100 + buckets: 20 +- histogram: chaotic_good_thread_hops_per_read_control + doc: Number of thread hops per control channel endpoint read + max: 100 + buckets: 20 +- histogram: chaotic_good_thread_hops_per_write_data + doc: Number of thread hops per data channel endpoint write + max: 100 + buckets: 20 +- histogram: chaotic_good_thread_hops_per_read_data + doc: Number of thread hops per data channel endpoint read + max: 100 + buckets: 20 +- histogram: chaotic_good_tcp_read_size_data + max: 16777216 + buckets: 20 + doc: Number of bytes received by each syscall_read in the data channel +- histogram: chaotic_good_tcp_read_size_control + max: 16777216 + buckets: 20 + doc: Number of bytes received by each syscall_read in the control channel +- histogram: chaotic_good_tcp_read_offer_data + max: 16777216 + buckets: 20 + doc: Number of bytes offered to each syscall_read in the data channel +- histogram: chaotic_good_tcp_read_offer_control + max: 16777216 + buckets: 20 + doc: Number of bytes offered to each syscall_read in the control channel +- histogram: chaotic_good_tcp_write_size_data + max: 16777216 + buckets: 20 + doc: Number of bytes offered to each syscall_write in the data channel +- histogram: chaotic_good_tcp_write_size_control + max: 16777216 + buckets: 20 + doc: Number of bytes offered to each syscall_write in the control channel + diff --git a/src/core/lib/event_engine/extensions/chaotic_good_extension.h b/src/core/lib/event_engine/extensions/chaotic_good_extension.h new file mode 100644 index 00000000000..10fec67b1d9 --- /dev/null +++ b/src/core/lib/event_engine/extensions/chaotic_good_extension.h @@ -0,0 +1,45 @@ +// Copyright 2024 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GRPC_SRC_CORE_LIB_EVENT_ENGINE_EXTENSIONS_CHAOTIC_GOOD_EXTENSION_H +#define GRPC_SRC_CORE_LIB_EVENT_ENGINE_EXTENSIONS_CHAOTIC_GOOD_EXTENSION_H + +#include + +#include "absl/strings/string_view.h" + +namespace grpc_event_engine { +namespace experimental { + +/// An Endpoint extension class that will be supported by EventEngine endpoints +/// which need to work with the ChaoticGood transport. +class ChaoticGoodExtension { + public: + virtual ~ChaoticGoodExtension() = default; + static absl::string_view EndpointExtensionName() { + return "io.grpc.event_engine.extension.chaotic_good_extension"; + } + + /// If invoked, the endpoint begins collecting TCP stats. If the boolean + /// arg is_control_channel is true, then the collected stats are grouped into + /// histograms and counters specific to the chaotic good control channel. + /// Otherwise they are grouped into histograms and counters specific to the + /// chaotic good data channel. + virtual void EnableStatsCollection(bool is_control_channel) = 0; +}; + +} // namespace experimental +} // namespace grpc_event_engine + +#endif // GRPC_SRC_CORE_LIB_EVENT_ENGINE_EXTENSIONS_CHAOTIC_GOOD_EXTENSION_H diff --git a/src/core/lib/event_engine/posix.h b/src/core/lib/event_engine/posix.h index d09e4d5040a..1f57fa1e901 100644 --- a/src/core/lib/event_engine/posix.h +++ b/src/core/lib/event_engine/posix.h @@ -22,12 +22,20 @@ #include #include "src/core/lib/event_engine/extensions/can_track_errors.h" +#include "src/core/lib/event_engine/extensions/chaotic_good_extension.h" #include "src/core/lib/event_engine/extensions/supports_fd.h" #include "src/core/lib/event_engine/query_extensions.h" namespace grpc_event_engine { namespace experimental { +/// This defines an interface that posix specific EventEngines endpoints +/// may implement to support additional chaotic good related functionality. +class PosixEndpointWithChaoticGoodSupport + : public ExtendedType {}; + /// This defines an interface that posix specific EventEngines endpoints /// may implement to support additional file descriptor related functionality. class PosixEndpointWithFdSupport diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 39f3d505ed3..35da52b9935 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -2207,6 +2207,7 @@ src/core/lib/event_engine/default_event_engine_factory.cc \ src/core/lib/event_engine/default_event_engine_factory.h \ src/core/lib/event_engine/event_engine.cc \ src/core/lib/event_engine/extensions/can_track_errors.h \ +src/core/lib/event_engine/extensions/chaotic_good_extension.h \ src/core/lib/event_engine/extensions/supports_fd.h \ src/core/lib/event_engine/forkable.cc \ src/core/lib/event_engine/forkable.h \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 7ef0084dfc4..1f4c698de8b 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1979,6 +1979,7 @@ src/core/lib/event_engine/default_event_engine_factory.cc \ src/core/lib/event_engine/default_event_engine_factory.h \ src/core/lib/event_engine/event_engine.cc \ src/core/lib/event_engine/extensions/can_track_errors.h \ +src/core/lib/event_engine/extensions/chaotic_good_extension.h \ src/core/lib/event_engine/extensions/supports_fd.h \ src/core/lib/event_engine/forkable.cc \ src/core/lib/event_engine/forkable.h \ From 0f77a5ad9ec97c146f7e1aad837950376bc98031 Mon Sep 17 00:00:00 2001 From: Xuan Wang Date: Mon, 11 Mar 2024 15:55:17 -0700 Subject: [PATCH 28/52] [BoringSSL] Update third_party/boringssl-with-bazel (#36089) Change was created by the release automation script. See go/grpc-release. Additional Changes: * Boring SSL started to [Require SSE2 when targetting 32-bit x86](https://github.com/google/boringssl/commit/56d3ad9d23bc130aa9404bfdd1957fe81b3ba498), thus added `-msse2` to fix some build failures. Closes #36089 PiperOrigin-RevId: 614822548 --- CMakeLists.txt | 4 + bazel/grpc_deps.bzl | 8 +- gRPC-Core.podspec | 2 +- grpc.gemspec | 2 + package.xml | 2 + setup.py | 5 + src/boringssl/boringssl_prefix_symbols.h | 22 +- src/objective-c/BoringSSL-GRPC.podspec | 1038 ++++++++--------- templates/CMakeLists.txt.template | 4 + templates/gRPC-Core.podspec.template | 2 +- .../BoringSSL-GRPC.podspec.template | 2 +- third_party/boringssl-with-bazel | 2 +- tools/run_tests/sanity/check_submodules.sh | 2 +- 13 files changed, 554 insertions(+), 541 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d5fc01ad30c..bb8cc07afae 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -234,6 +234,10 @@ set(gRPC_USE_PROTO_LITE OFF CACHE BOOL "Use the protobuf-lite library") if(UNIX) if(${CMAKE_SYSTEM_NAME} MATCHES "Linux") set(_gRPC_PLATFORM_LINUX ON) + if(NOT CMAKE_CROSSCOMPILING AND CMAKE_SIZEOF_VOID_P EQUAL 4) + message("+++ Enabling SSE2 for ${CMAKE_SYSTEM_PROCESSOR}") + set(_gRPC_C_CXX_FLAGS "${_gRPC_C_CXX_FLAGS} -msse2") + endif() elseif(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") set(_gRPC_PLATFORM_MAC ON) elseif(${CMAKE_SYSTEM_NAME} MATCHES "iOS") diff --git a/bazel/grpc_deps.bzl b/bazel/grpc_deps.bzl index 3b7187bb91f..02a583535ff 100644 --- a/bazel/grpc_deps.bzl +++ b/bazel/grpc_deps.bzl @@ -240,11 +240,11 @@ def grpc_deps(): name = "boringssl", # Use github mirror instead of https://boringssl.googlesource.com/boringssl # to obtain a boringssl archive with consistent sha256 - sha256 = "057f662b0e85931a84945b2e89ba201fd44b0583da827c948fe443593690fb83", - strip_prefix = "boringssl-ae72a4514c7afd150596b0a80947f3ca9b8363b5", + sha256 = "06ba43ff1825c8a9a45dae7f85e532153a531707f6a3e56be1e892fd2d3b75f6", + strip_prefix = "boringssl-e14d29f68c2d1b02e06f10c83b9b8ea4d061f8df", urls = [ - "https://storage.googleapis.com/grpc-bazel-mirror/github.com/google/boringssl/archive/ae72a4514c7afd150596b0a80947f3ca9b8363b5.tar.gz", - "https://github.com/google/boringssl/archive/ae72a4514c7afd150596b0a80947f3ca9b8363b5.tar.gz", + "https://storage.googleapis.com/grpc-bazel-mirror/github.com/google/boringssl/archive/e14d29f68c2d1b02e06f10c83b9b8ea4d061f8df.tar.gz", + "https://github.com/google/boringssl/archive/e14d29f68c2d1b02e06f10c83b9b8ea4d061f8df.tar.gz", ], ) diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 1c01a1e54c0..8326e6b78bd 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -195,7 +195,7 @@ Pod::Spec.new do |s| ss.libraries = 'z' ss.dependency "#{s.name}/Interface", version ss.dependency "#{s.name}/Privacy", version - ss.dependency 'BoringSSL-GRPC', '0.0.32' + ss.dependency 'BoringSSL-GRPC', '0.0.33' ss.dependency 'abseil/algorithm/container', abseil_version ss.dependency 'abseil/base/base', abseil_version ss.dependency 'abseil/base/config', abseil_version diff --git a/grpc.gemspec b/grpc.gemspec index d5eca0ba3cb..8b33f403ed2 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -2845,6 +2845,8 @@ Gem::Specification.new do |s| s.files += %w( third_party/boringssl-with-bazel/src/include/openssl/pkcs12.h ) s.files += %w( third_party/boringssl-with-bazel/src/include/openssl/pkcs7.h ) s.files += %w( third_party/boringssl-with-bazel/src/include/openssl/pkcs8.h ) + s.files += %w( third_party/boringssl-with-bazel/src/include/openssl/pki/certificate.h ) + s.files += %w( third_party/boringssl-with-bazel/src/include/openssl/pki/signature_verify_cache.h ) s.files += %w( third_party/boringssl-with-bazel/src/include/openssl/poly1305.h ) s.files += %w( third_party/boringssl-with-bazel/src/include/openssl/pool.h ) s.files += %w( third_party/boringssl-with-bazel/src/include/openssl/posix_time.h ) diff --git a/package.xml b/package.xml index 901a73b61b0..cfe611d7459 100644 --- a/package.xml +++ b/package.xml @@ -2849,6 +2849,8 @@ + + diff --git a/setup.py b/setup.py index 7c034fe3eba..93138a2d11a 100644 --- a/setup.py +++ b/setup.py @@ -393,6 +393,11 @@ if BUILD_WITH_BORING_SSL_ASM and not BUILD_WITH_SYSTEM_OPENSSL: if BUILD_OVERRIDE_BORING_SSL_ASM_PLATFORM else sysconfig.get_platform() ) + if "i686" in boringssl_asm_platform: + print("Enabling SSE2 on %s platform" % boringssl_asm_platform) + EXTRA_COMPILE_ARGS.append("-msse2") + else: + print("SSE2 not enabled on %s platform" % boringssl_asm_platform) # BoringSSL's gas-compatible assembly files are all internally conditioned # by the preprocessor. Provided the platform has a gas-compatible assembler # (i.e. not Windows), we can include the assembly files and let BoringSSL diff --git a/src/boringssl/boringssl_prefix_symbols.h b/src/boringssl/boringssl_prefix_symbols.h index 368614cbed7..827f1b0ed1c 100644 --- a/src/boringssl/boringssl_prefix_symbols.h +++ b/src/boringssl/boringssl_prefix_symbols.h @@ -1,4 +1,4 @@ -// generated by generate_boringssl_prefix_header.sh on BoringSSL commit: ae72a4514c7afd150596b0a80947f3ca9b8363b5 +// generated by generate_boringssl_prefix_header.sh on BoringSSL commit: e14d29f68c2d1b02e06f10c83b9b8ea4d061f8df // Copyright (c) 2018, Google Inc. // @@ -317,6 +317,7 @@ #define SSL_generate_key_block BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, SSL_generate_key_block) #define SSL_get0_alpn_selected BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, SSL_get0_alpn_selected) #define SSL_get0_certificate_types BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, SSL_get0_certificate_types) +#define SSL_get0_chain BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, SSL_get0_chain) #define SSL_get0_chain_certs BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, SSL_get0_chain_certs) #define SSL_get0_ech_name_override BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, SSL_get0_ech_name_override) #define SSL_get0_ech_retry_configs BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, SSL_get0_ech_retry_configs) @@ -1285,12 +1286,17 @@ #define ChaCha20_ctr32_ssse3_4x BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, ChaCha20_ctr32_ssse3_4x) #define DES_decrypt3 BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, DES_decrypt3) #define DES_ecb3_encrypt BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, DES_ecb3_encrypt) +#define DES_ecb3_encrypt_ex BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, DES_ecb3_encrypt_ex) #define DES_ecb_encrypt BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, DES_ecb_encrypt) +#define DES_ecb_encrypt_ex BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, DES_ecb_encrypt_ex) #define DES_ede2_cbc_encrypt BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, DES_ede2_cbc_encrypt) #define DES_ede3_cbc_encrypt BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, DES_ede3_cbc_encrypt) +#define DES_ede3_cbc_encrypt_ex BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, DES_ede3_cbc_encrypt_ex) #define DES_encrypt3 BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, DES_encrypt3) #define DES_ncbc_encrypt BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, DES_ncbc_encrypt) +#define DES_ncbc_encrypt_ex BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, DES_ncbc_encrypt_ex) #define DES_set_key BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, DES_set_key) +#define DES_set_key_ex BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, DES_set_key_ex) #define DES_set_key_unchecked BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, DES_set_key_unchecked) #define DES_set_odd_parity BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, DES_set_odd_parity) #define DH_bits BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, DH_bits) @@ -2551,14 +2557,9 @@ #define X509_PUBKEY_set BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_PUBKEY_set) #define X509_PUBKEY_set0_param BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_PUBKEY_set0_param) #define X509_PURPOSE_get0 BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_PURPOSE_get0) -#define X509_PURPOSE_get0_name BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_PURPOSE_get0_name) -#define X509_PURPOSE_get0_sname BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_PURPOSE_get0_sname) -#define X509_PURPOSE_get_by_id BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_PURPOSE_get_by_id) #define X509_PURPOSE_get_by_sname BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_PURPOSE_get_by_sname) -#define X509_PURPOSE_get_count BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_PURPOSE_get_count) #define X509_PURPOSE_get_id BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_PURPOSE_get_id) #define X509_PURPOSE_get_trust BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_PURPOSE_get_trust) -#define X509_PURPOSE_set BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_PURPOSE_set) #define X509_REQ_INFO_free BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_REQ_INFO_free) #define X509_REQ_INFO_it BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_REQ_INFO_it) #define X509_REQ_INFO_new BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_REQ_INFO_new) @@ -2665,6 +2666,7 @@ #define X509_STORE_free BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_STORE_free) #define X509_STORE_get0_objects BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_STORE_get0_objects) #define X509_STORE_get0_param BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_STORE_get0_param) +#define X509_STORE_get1_objects BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_STORE_get1_objects) #define X509_STORE_load_locations BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_STORE_load_locations) #define X509_STORE_new BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_STORE_new) #define X509_STORE_set1_param BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_STORE_set1_param) @@ -2677,13 +2679,6 @@ #define X509_STORE_set_trust BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_STORE_set_trust) #define X509_STORE_set_verify_cb BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_STORE_set_verify_cb) #define X509_STORE_up_ref BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_STORE_up_ref) -#define X509_TRUST_get0 BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_TRUST_get0) -#define X509_TRUST_get0_name BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_TRUST_get0_name) -#define X509_TRUST_get_by_id BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_TRUST_get_by_id) -#define X509_TRUST_get_count BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_TRUST_get_count) -#define X509_TRUST_get_flags BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_TRUST_get_flags) -#define X509_TRUST_get_trust BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_TRUST_get_trust) -#define X509_TRUST_set BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_TRUST_set) #define X509_VAL_free BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_VAL_free) #define X509_VAL_it BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_VAL_it) #define X509_VAL_new BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_VAL_new) @@ -2782,6 +2777,7 @@ #define X509_getm_notAfter BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_getm_notAfter) #define X509_getm_notBefore BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_getm_notBefore) #define X509_gmtime_adj BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_gmtime_adj) +#define X509_is_valid_trust_id BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_is_valid_trust_id) #define X509_issuer_name_cmp BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_issuer_name_cmp) #define X509_issuer_name_hash BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_issuer_name_hash) #define X509_issuer_name_hash_old BORINGSSL_ADD_PREFIX(BORINGSSL_PREFIX, X509_issuer_name_hash_old) diff --git a/src/objective-c/BoringSSL-GRPC.podspec b/src/objective-c/BoringSSL-GRPC.podspec index a1f9c545f8d..4103011d3bc 100644 --- a/src/objective-c/BoringSSL-GRPC.podspec +++ b/src/objective-c/BoringSSL-GRPC.podspec @@ -39,7 +39,7 @@ Pod::Spec.new do |s| s.name = 'BoringSSL-GRPC' - version = '0.0.32' + version = '0.0.33' s.version = version s.summary = 'BoringSSL is a fork of OpenSSL that is designed to meet Google\'s needs.' # Adapted from the homepage: @@ -76,7 +76,7 @@ Pod::Spec.new do |s| s.source = { :git => 'https://github.com/google/boringssl.git', - :commit => "ae72a4514c7afd150596b0a80947f3ca9b8363b5", + :commit => "e14d29f68c2d1b02e06f10c83b9b8ea4d061f8df", } s.ios.deployment_target = '10.0' @@ -175,523 +175,523 @@ Pod::Spec.new do |s| *) opts="--ignore-garbage" ;; esac base64 --decode $opts < src/include/openssl/boringssl_prefix_symbols.h - H4sICAAAAAAC/2JvcmluZ3NzbF9wcmVmaXhfc3ltYm9scy5oALS9XXPbuJaofT+/wnXm5kzVrpnYidPu - 986xlY4mju0tKT2duWFREmRzhyIVgvJH//oDkJSIj7VArgW/VbtmOpaeZ1EAiC+CwH/918mDKESV1mJ9 - snw9/iNZllVWPEiZJ7tKbLKX5FGka1H9p3w8KYuTT82n8/nNyarcbrP6/ztJxW9n6Yfz0w+r39LN+vT8 - 3fnvH5fv0ot3v3/4bfN+lf6+vHj/8f3y/N/+7b/+6+Sq3L1W2cNjffJ/V/9xcvbu9OIfJ3+U5UMuTqbF - 6j/VV/S37kW1zaTMVLy6PNlL8Q8Vbff6j5Ntuc426v+nxfq/yupkncm6ypb7WpzUj5k8keWmfk4rcbJR - H6bFq3bt9tWulOLkOavVD6ia/1/u65ONECcKeRSV0L++SguVEP842VXlU7ZWSVI/prX6P+IkXZZPQptW - x2svyjpbCX0Vbdxdf72Hj3Y7kVYnWXGS5rkmMyEPv27xZXIyv/u8+J/L2eRkOj+5n939Ob2eXJ/8n8u5 - +vf/Obm8vW6+dPl98eVudnI9nV/dXE6/zU8ub25OFDW7vF1MJ3Pt+p/p4svJbPLH5Uwhd4pSvt59e3Xz - /Xp6+0cDTr/d30xVlF5wcvdZO75NZldf1F8uP01vposfTfjP08XtZD7/T+U4ub07mfw5uV2czL9oj3Fl - nyYnN9PLTzeTk8/qX5e3P7Rufj+5ml7e/ENd92xytfiHUhz+S33p6u52Pvnnd6VT3zm5vvx2+Ye+kIY+ - /LP5YV8uF/M7FXemft78+81C/4zPs7tvJzd3c33lJ9/nExXjcnGpaZWG6pLn/1DcRF3gTF/3pfrf1WJ6 - d6t9ClChF7NLfR23kz9upn9Mbq8mmr1rgMXdTH33+7xj/nFyOZvOddC77wtN32lnU4Tvbm8nzXfa1Nfp - oa6luYrJTCXEt8tG/NnOjf9syv+nu5lyqtsnuby+Tu5nk8/Tv052qayFPKmfyxNV9Io622SikqrwqMJf - FkJlQq2LmCrUW6n/oEVZre9WXeLKzck2XVXliXjZpUVTCNX/slqepNXDfqt88mQpFCyaQOru/c9/+/e1 - urMLAV7O/03/cbL8D/CjZKp++qz9QtBhfvEkPfn3fz9J9P9Z/ltPTe+STaJqGfga+j+2f/hHD/yH5ZCi - plo6pPdcL27mySrPVFIlW6Gqh/VYnU86VoYO9EhRPYmKo7NIx6rrwmS532xUceO4Ad6O8HSanPFT1qcB - O1OL+tgp7dOePSYlwunwoMp0nW2FbtloXoP0rI+qhcsFU2zDnpuVCMivj8mzcI7puiIrsjpL88MvSdb7 - rualBsJVfdzJbJb8MVkkN9NPY/0G4ntmk8u5aqmIqpaybXmZrhP9Zd3nUh1EitNle/Pd/eRWf6BThlKR - u1xvvJ98SyrRxZurTsx0/O+HWMC8zMoou8PbEZ4r1bZz9R4MuSMuHxT0MfQfr6b3qj+VrIVcVdmOcqPA - NGjXtVa6V61Pka0ZehNH/Uvdh+K5NYp6V9lOjToirrwXoDHW2YOQdUSMXoDG0BW8fEx/iu7LzEiuBo3H - /i2B3/DzJSnSrWCKOzpoZ191C6PubfqSqIZL8u4vx4BHyYrYKL0BjRKRBcH031WbiAzo6IC9rMtVmScR - EY4GNEpc6odSPpNJqlojhrkjMesyL1c/u1qKZzcNYBRZq1ojrdbcomPxToS7b/dJul4nq3K7q0QzrUPs - Wg5ogHibSgjgm5IcERMBMVX5eEdPP4uErW/yQxAPEjFbswJka8THTRYoVRZ/6XLwLlk9pqouXImK1lL6 - OOg/jfOfDvmbT6wcSfMHRiDQg0Rsh7xXl6wwBxh2i5e6SuOSzHPAkWT7MzkBOtT3rh6Fqh93VfakZ+x/ - ileq3RMAMdpepvptD1W535Ej2Djgz0VaGaknyRFcARbDzSdmJE+DxduWa8ELoUnMWjajIea1d7DvFkW6 - zEVSruRON4q7XA3PqSEgBxpJZg+F6GoBPQ2igO1OMkPCMjR2nUudf0UhyJ02TOLH2uR7+Xi4dck/zKYB - u2rfyU7F+KamEdcpl22ylaoFqFaXxyLo+4Xn1mTIyruZXR6JsEurdMtyNyRmbWtcRo3t4KC/vRFkrZ/1 - 0PUGjdibKl2y1C2KeA9NdZJnsmbpLQMcRf0p3edq0JVK+azqjCUnkCcZGSvZS1Gt0zp9k6BHGxxdvCTc - UB2KegvxrJr0tXhhyo88FiGypQYlcKys2JTJKs3zZbr6yYljCeAY6kbNy4eoKI4CjqOncpq7l3sDWQI8 - RjNhwZqSwCRILJV18bFcCRKL0Vs7cLCx2G9Vb2T1U/DKr4HDfmZP0EBh7699ph+NP+7rdfnMSnLbAEdp - noCkj9SZJ4+G7V3PSd0vaojDzlvfAkcjPhkFUMSbS1WLdaVAVwGszPYtcDR1e2Sb16haylEE46zFrn6M - CNLwwQjcbDdw3988w+y+kZerlHUPghI/ViHUqKbe7pLZnDz5YbKQ+ZkufPY9ldiWT4I7uWHTvl1/kKSr - lcppqtpAg97koSzXEfKGD0eoRCEeyjpjDK4QDRKvraY2+zxnxelxzL9MHjN6Y2aymLlU4+gVL5M7Nmzm - Z7MpGIgRm9GAB4nYDHaa7JLZ37xgtiIQp/nikh2jxQN+PRaI8Ld4wN9VMhEhjgYkCvumCNwReiGx4Flb - FPGqXuWS+DjORhGvjC+RckyJlHElUg6VSBlXIuVQiZTRJVKOKJFdr5JXfg4w5K7fdQs9k11ZMpoZm0ci - sOYKZWCusP3sMDkkeeojjvgPfV/23BtsAaOdstPoNJBG6rN99cSpdY5o0MualnB5JIJYPbIGSBaMuJsn - V0m25smPdMgeoQ57+Wlu8EgE1tx4TyJWmT2k+QMvQTo2bOYniSlAYsQ9WwIUSJy3qG1OR9Y2iRrOl8/J - vvhZlM/6Qf2um1HjZBIuw2JHRhvjlyLXHW9Oi+wa4CjtageWvkMDXm7+D+Z783nktBDmQSI20/Vpseas - ZvAESIx2SQKzFjBxxB/1HEuOeI5lfCemYFkGJEq53eVZWqyE6rDl2YqXJ64EibWvKn1Buv/J/Um2Aouj - ivy2K4+8KIYAjhH9lFGOe8oo3/QpoyQ+ZTS/393eu7R+lDFxTQ8SsZRNja7q22Zynpe2rgSOJdIqf22e - hXbrPjhNOmBBovGe2MrQE1v94SbNpdBrcqqu+RXrpHsBumm9OAGHnPCVPFQiVVhEWtoGOErUM105/ExX - xj/TlWOe6crYZ7py+JmufItnunLcM93D16RQ7fOmSh/0a8ncWJYEiRX7/FiOe34smc+PJfr8uPlExhUv - kx+OkKTVQ2wU7YAjFfoJZJuKUX1tyDMUUSbp+kkvUJNiHR3WkSGx+U/+5dCTf/2FZollJeSuLCSr0FkC - JAZvdYEMrS7QH+pNMva10MtzRCG5IXwLEq1f2sx5eQO1INHkz2OvOuLGBTR4vO7F5dh4jgaJ122iwonR - orD31z5bRWSPgaP+iBUtcsSKFhm1okUOrGhpP1+V1bp/VyyiRUNUWNxaj6jLQvVg5WN6dv4xKTfm2FHy - LmHIil1NNz5QfXZVf+23ghfdtcDRDk1Mv7qZ2X6AIixm7MolOXLlkvm9TL+gVtSqOo2J1lvC0XSFs34U - 3HVTARUSF3o/gN2hxm149Kx40C84lZUaIW2bHbUkNzSgQuJW9U7f5JssF7xopgCJUVfZKnpKzbfA0bol - bPql04jmwrdg0dilM1ga7fn9mLEwbEKj6k5s287r1xO5HX5QNDZmTDcFt4Wj12m9l7G/9igZE4vXSLiO - YKR+NWdcNMszMqJ8k3gyGG2vJ5dU/RMR6qBA4qg6e/3I0jdkyBpXzG0FHkes+NevWdxcyZQrVmjQG500 - pgOJVO15zVADwk7+w4LQU4KuF/oGHQPYFIzKWn8tB9df7/XEwobqbSnApu7h+3b0/ZX+QNCmh+zJ5fz2 - NC5EoxiMo/tTkXG0Ao4zm1/GJZglGBGDnWy+ZUw0buL5FjhaxKuwDj7oZ6ec6xiO1D4W56YdbBqO+hbx - 8Eh66NdulFq/Jo8Z/UkCKLFjTa6+JF8nP+Z6HwaK3uQQI/UVbgtEnI+pTNb7Xd5lVVlssgfiMqQhFxJ5 - m1byMc31xE712n1bsuKCJiQq8TUWk0OM9ObLQW1vtzVeojeNPj4e7R8HU+IMqOC4xpPnVbrTw0NOSN8C - R6MWaZPDjOU2Wb7WtAkMn4bt7R4A5A2qADzg502tIYpAHPZDIdwSiLYTEWmm4QG32QbIqECWaShqOxcd - F691BCK9zXTkSGXgOtqxODtmi6N+zmoWAA/6WfsQYA48Eq0FtUncutX7vVfUhY6wAY8S88Ao5MEjdlM8 - ebYRzTo8atdsyBWKvBX8SFsRNhPnggEc90dmTjBPdEcusnJzFHgcfpXS07A9k+2jOm4fxuThCMTOpIHB - vmaFPa/q6NCgN6ZX4SjQODF1uByqw+Ub1U5ydO3UP/3hxgmVUBlRA8lgDSTjaiA5VANJNZbI18lSv3lZ - PORCj4xZgQAPHLEu+b36Axs2J5uyishsQAPHow8YbdK20jc7gPY4iNhnNLjHaMT+osG9RfUml+munWrQ - D/VVga0pZwuEHH4kvW19++bLfvkvsaqlzmzVYaY9kwib/KisXUwDO5jqj/Tc2Bv9lIDKiZvrL+mN+btT - HEiRXHjAneRlZIDGAEVp5ga6Rxm6Y5DX9Di+A4pUv+4EO60MeMDNTCvXYEdp1w89ZqTEOUKuS6+2ypvl - +8w9axGFE0cvH2s3PCW5e8zxxeyyO7DDLv0qgeuL2UF3YPdc3k622C627B1sA7vXMraOAXeMWe3r+rEq - 9w+P7ftqgvb8B8Bt/1oV2wd9ymKyqkTzwCHNdf+IND5AJU6ssj9Og6Q3OMeoOiuMFxoNzPa1M8rH9wZW - 9Uu/lFuPaClBhlxQ5GYuu+060XIAwFG/flNJ90TIVT/mcCKtHnk/weAcY+Qu0MM7QL/Z7s+EnZ+jd30e - seOzqCo1TmAeduTBjvtlV1bNkindRm/V7V+p254UADTYUajPbvxnNsejY/VisuboDorPp117/c581Z5W - 5n0asJuPnXW3SJIjeAYoCq+hDu9X3Xyqb+xmXWSp+qRVRmuzYQMShf2UFzYAUYwXvY6bodFzHLQA0djP - zoaemfH2EMf2D++fMcWOlsMmLCr3mdyYZ3H9d7pOTncmSLuejRkOVGFx3TV0zJieBojXvW1ViV971WSp - Boy4KxUqAWPFvOKBKKA4b/JUk/Q086HZlIe+96jJecakWx5EFB4w36c6psez+lTdSs1oj0ci6C2yIgL0 - OOxvt7Fi+w0c9us8T+t9JYxFrOxoqAyJfTgGLDabQBEcs3tQwY9lCfwYzHWMDgp421+2fE2e0nxPd9s4 - 6mfUG/j7Q8xTK9ATK+JOqxg6qcL4vFLFqdwy5S0MuLtNcugLn3w6YO+P9mKH6BV4nP64e2aUowCMoSrF - bM1QNxxmpB4rZ5O+9bB3DuMZIYD7fm8+ghrBEwAx9CCY7NUQ4KI/tUZXHBkfJH+dv/s9mS/uZpNm/XC2 - fmGGAExgVNb6pvC6pu5olK1M5H6npwXoagP23Rvy3bIB7hP1j0w+Crqr43zjYRtOqvHAYUbOvdyTvpW9 - d9HAWTTNx0/k9k8hvuc4RZPkglwXWLDvZu93NHB+TfTZNSPOrYk+s2bEeTWcs2rgc2ra3dMPsyL04x0h - 3o/AeNqDnlDTrEM8TCPQt0AG8ICf2Xl2eSQCt4KzYMy91wO6uCRyHEikZueVWnU0ZTPB3ExZSVY80IRE - BUZ3rJiAB4pYrPWsOa+3bNOAnXUQoE0CVuOlJrLXYMNm8sJeUODH4O/WM3T2VHOYwzIrqU7NACbWfj+h - 06uOn0k9p1esBEt8gAE3vXNWQb0zKVb6runPKWkmj3ndyZALitw+vbH2JqGHBCRQrHZ+lTUGt2DUrV9o - Z9z7No3ZOT3TngxZm2dbfHWDQ37WbAE6jysf00qsuRM/No3aGbvV+zRk59V+eL0HTYmuswdB72TjpnFR - 9QCAVYACrnGRWXcE4gEicvdbegjvtWS8B5M+iET+pL2nAOCAn704wqdh+77IftGni3sStBr75RwfwjJC - QJqheJwS7Bv8KBHb7Q+ewBhz+mL45MWIUxeDJy4aH9IX6Xow6Oa0OejI/JnRu3wGe5fP9L7aM9RXe1ZV - lmB3KG3atus3tmLXIWAOP1I3kqLKO8z2ZQXzHXwL9JzGluhEqUF6VjXWp+o04nhksla1D8nTIp5Hy1nT - Fy7rmdseIlHZQr4LaLb11lE7SU2EgMmOqvsi+92aOGfUU7Ytz5ZVWr2Ss9/kHKM+dLZ/8EgdOQE44G/X - MrbLVSVZb9G2fZs+ZKvjfMpx+8+aVF5QiRur3YJEL1Rrl6jRgri0a9eb16sv6EV21OkDD7bd3BOD8dOC - iW/Fem/D6s3MrcE9qVT4tG3fCUHqIunvuwZyuwK2KarvvtKnJzYTmbtS1rwl+AENHE9V0afvm4d9h+JM - f+lxyOVFfsrWor1Eagvqwba73cpblfHjr042efbwWFOfNAVFQMxm5iwXTyInR+lRwNt2oHhig7XNFbHS - qLx6gnlUMXoysfEB544CcNffLHI0clPPHUtaDFDhxpHucoV/Ed8uQhR2nG5D8H59MiWCB7tufTCKipy3 - r/jR1DbrmvV7A9nfot0GKsuzOqNNdcAGLEpEbqMSN1Zbz1WC+iqWTbpWzim22Am2EafXBk+ubT6kPg45 - QoAr6kzKMaffNt955lzxM3TFp6w8OkXyiHN6LnpybsypueETc5tPofcIySEgCRCr7wbzfonDAxFY5/OG - zuZlnsuLnskbcx5v+Cze5tPHkqHUEOAiv6uCnefLPcsXP8c36gzfgfN7I8/uHTy3N/7M3jHn9Ure2wsS - e3uhOd22eVO0mbOmXq/FAmbeyb7BU327D2Wzt6seyKzKtdiVxIUKuMWPRm+NEqgt4hzkip4OHHWS7sAp - uhEn6AZPz407OXfo1Nzos2xHnGPbfqXZWoB3u1gw4OaeWztwZm38OadjzjhtvtO+SK1b9PYYT3IQVwDF - 2JSVyiE9RdvMrcr0gREHkACx6OvM0V3RJHnttATWTuu/RY2a6qHxUt30HDZ5+kA3H0DfyV71PHBaq/74 - X+ufp6fJc1n9TFU3qiCnscv7EdhrlgfOZ40+m3XEuazRZ7KOOI81+izWEeewcs5ghc9fjTl7NXzuauyZ - q8PnrTbfqPdkab33PeyX4gdOGGWeLoqeLBp/quiYE0XjTxMdc5LoG5wiOuoE0Tc4PXTUyaHMU0PRE0OP - x32aW9LT32oPaJB4vOxGTyY9fhizeB6VILH0aEZP2axe+cMiVATGZK5kHDpxlX/aauik1faz/kEEpzVx - eSjCW56nyjlLVdJXgktoJbjkrdmV2Jrd+PNIx5xF2nznUayNfi79ET8qgWLxyj9e8t9mow3KSaZvdIrp - 6BNMo04vHTi5tD1vlDE6R0blcSegjjn99G3ODB17XqhxgKIer5HXTEM8GiFm7a4cu3ZXRq/dlSPW7kae - XTl4biXvzErsvMrIsyoHz6nknlGJn0/JPJsSPZcy9kzK4fMoWWdRIudQ8s6gxM6ffJuzJ8eeOxlz5mT4 - vElJXyctoXXSrDYabp/JLQvQqug/MXYNNTncSN4m2oNtd13WzWFt3BV+EG9H4J8BGjr/M/Lsz8FzPyPP - /Bw87zPqrM+Bcz7jz/gcc75n/NmeY871jDjTM3ieZ+xZnsPneMaepjl8kmb0KZojTtDUq6OSR5HnZbfn - Z7cOjxgGdNiRGPPK4Ezyc0pLBP191yD7x0ZJVjylOW09AShwYujFoSSnBizH09n7wzQBeXrLYz0zS4m4 - ujlGltJie/PiZs778R5oO+kyyML6wR5oO/WZoclyv9moQs8wA7jlfzpNTtkp6sO+myfFbNwU9mHXfRaT - CmfhVDhjSjFbRCqchVMhIg2CKcARwqaI34788vVZlhgnPI11Ohjqo6w1AtDem52tOdfpYKiPcp0A2ntV - z+Jq9uN+cZd8+v7582TWDLTbA5A3+2I1NsaAZiie3un+DeIdNYF4ayF2zYWxQx0NgSh6RVuxz3N2kIMg - FGO/5ev324B5t5ePbLWGA245/r0piA2YSZvlwrRln88W9+r7d4vJ1ULfN+o/P09vJpy8HVKNi0vK74Bl - VDRiGQhp7Hh6Fez0/suxjtjuqHc+psDi6FX0teAFaFnUPH47Pw/EnOpPa55Uk5iVU2h9GrXTiqYFYk5q - AbRJzEqtJFzU8jZbzN5efpuwizJiCEZhtM2YIhSH0yZjCiQOpy0GaMROvJFsEHESXtV2OdxIvTF9GHOT - bkuLQ4y7ckc6xgiEETetZ2BxuDHupjQFWAzChnweiDiplZRD+ta4G3roXuYWYbz0MgouWGa5xRUvqfIx - 25Dzu4F8FyubnRy+vLpSw7rkejK/mk3vm64X5QcjeNA/frMUEA66CfUrTBv2yTy5+nZ5NdrXfd82rJar - RBSr6nX8kdEO5vg2y9OzC5bSIh1rXXGtFmlb14Ks6xDbI1ZLzqUZmONjuCBPyc6LMpAXsjnuofmA8l4Y - gPreLiDHa6C2d188V+mOquwpzJbs0vV6/AIqELbdnOuErzLiGvErnN+eJpe3Pyj1Y484nk/TRTJf6O+3 - xxuTjC6Mu0lNBcDi5ofmJcyaK+9w3M9Xh6yU5sdHA979Nlm+Eo70QwV4DEL3GUCD3piclHBOfrtnF0EL - Rb3UKzZA1EkuHibpWu/ubiaXt+TrPGKOb3L7/dtkdrmYXNOT1GFx8wOxjNlo0JtkRf3xQ4S9FYRj7KOD - 7AeiZOwECuUoteDZKO6V/PyUofyUsfkph/NTRuenHJGfdZl8uuUGaGDH/Zl5439G7/w/Jrcq3s30fyfX - i+m3SZKu/0UyA/xABHqXBDQMRCFXY5BgIAYxE3x8wE+9cQF+IMKuIiwoww0DUagVBcAPRyAuyB3QwPG4 - vQ4fD/p55QrrgdgfM8sU2hOZXp5zU8VGUS8xNUwQdVJTwSJd6+1i8od+mrjd0Zw9hxgJDwhdDjHS88gA - ESe1W2dwuJHRAfDogH0fp9+H/BkvOTIsNchltecQo2TmmERzTEblmBzIMRmXY3Iox+jdNIt0rLffb27o - N9qRgmzEItUxkIlamA6Q47r79N+Tq4XeV5CwZN8nYSs57QwONhLT70jBNmoa9pjru1pM+sk2YvPhwiE3 - tSFx4ZCbnlsuHbJTc85mQ2ZyLjpwyE2tYF3Ycd+rvy8uP91MuEkOCQZiEBPexwf81OQHeCxCRPoEU4ad - JoHU4KcDkALzyT+/T26vJpwHCQ6LmblWwLjgXeYCucK2WLRJk67XNKsDh9yrXKQFsT6FBHAMaiuA1v+H - Dwjro1wONlI21HM5xMhLzTWWhuTbH68V+wdK79g//Aij7kT9Od3neps2+ZMZwnLAkXJRPIx/u9snYSu1 - AkPr7+4D+pSUCQaciXhhaxUbNiebXYxc4bCf2pNA+xD9B++YwneoMVm+JrfTa6a3o3F77N0hR90d7reS - VK7eIpr2wBHV4PH74vMFJ0iHIl7C7ikuhxu5N/qBdcyLj6fc6tpGUS+xZ2GCqJOaBhbpWpnPchbosxzW - AxzkqQ3zUQ36fKb5YJ1tNnSdpiAbveAgz3U4D3PgJzisxzbIsxrmAxr0qQzrUQzy/OX4tGRXyuyFZWxR - zMt4mBN+guN82iyHjdE3AiiGqpofRCGq5nCbtd61jR7GdyCRmMl/IBGrDpjULG2Lut4f9xPyyOYAQS76 - nX+gIBv1AcYBglzke7+DIJfkXJeEr0ufTsGSnTq277fTPyezOf9ZKCQYiEGsmn18wE/NNIB3IyyuWI2x - wSFGepNskZh1u+Pc9T6O+OmlxAARZ8a71gy7RnIp6DnESG+8LRKxUqsFg8ONnAbXxz3/5wt2NWGzuJlc - DAwSt9ILg4k63j+n82nE7L2PB/3EBHHhoJuaLB7t2NfZA2GrKQNxPG1vqRbJ03uSzOA8Y52US8rZkg7m - +LJabJP1WUayHSDERdnHwwMxJ3Eiy+BAIz2DDQ407jkXuAevTh/0wsmSlkOM5PvbBBFndrZmKRWHGKl3 - ssFBRt6Pxn4x6+civ1VvYMO6TzoQc3Luk5aDjKzsQPJilxJ7iEcKsukNwek2TWG2ZFW/8IyahKz7gveb - Ww4y0vbydTnHuF12cwbkp3EWiVkLvrYAvG3zpdL7b9odbXCOUfVmt1mdPQl6NWGjrndfJ6KkzdJ3DGBi - tPY95vjq9OGM+tpTxwAmlVlkk2Jck9ju8mafUWomWKRh/b74ooDFj2R6+/ku6V6pJtlRw1AUQtoi/FAE - So2MCaAYXyc/ptfMVOpZ3MxJmQOJW1mpcUR776fL+fQqubq7VUOCy+ntglZeYDpkH58aEBsyE1IEhA33 - 9C5Jd7vmeLYsF5QDHQDU9h5PIlvVVU6xWqDjzEVaJaQTBh0M8rUbBzOtBuy49WZFhT61ofkKyWyjjpea - nH4qqr80w8XmuCPipsuoAInR7C2cPOzTKi1qIVhhHAcQSZdDwiSSy9nGdXk4b5Xi6ynbJsoNRaO+bvN6 - VyfSg3ULclw5YXOyI+A4KlouOvVk95ckzXOqRTO2qVl9RFgcZTK+iXhmq4OBPr1VkMqK8et/INY3jz/Y - oicAy45s2fmWrMhqqkczvmmrp0sYGXDgYONufBfWwXwfOzsDeclsfRwU8+qjkMdvfA+xvpl6JorLeUbq - D3d+7aN4We+3pMLcIbZHZ1BBKsst4Vpqcht9YGyTLobNQXUFLYVMzjXWj+QK/AgBLkpX1GAAU7NlHeml - HgDFvMTssEDEuVZdnqp8ZWk7FjFTbwgLRJy7PdOpQcRZEQ7Y9EDESTq6wid9a0nvOxmY7SMWdq+c60Zg - mZXJLs0qoujI+UZGV9XAfB+tb9ESgIVwIo3JAKYd2bPzLbpOXO43VFWH+T5Zrn4KcqK3lGt7IXpeXMN+ - uxQV+X40MNCn7yjVhjCUHWlbGUM0cHS2K0kFQn3d4fUCB1JBaAnHUlfkZuXAOCbikGznjciolbtfp1OL - jl9m2pOTZXFK1TQQ4OLMR1mg65S027UBHMcz76qekWuSnLpbwjW3JNbb0qu1JbnOlkCNrc//2dIkCnAd - 9NpVgnWrFOInyaK+7xpULzAnnFFvQYBLZV5z+i21FHkw4tZDiR1hb2cQRtxsL+ykjvUlOHMjeTM3Epu5 - keT5FQnMrzR/o47pjxDg2pFFO99CnauR4FyN7KZIiP0pA4N9otzomYd9VXC0Pe3bC8IyDJPxTceZEXIJ - 6cmAlThXI4NzNf2ncidWWZrz1B2MuclDNgf1vZz5JYnOLx0Hh90JdaTlBajAifFY7vN1osZonJR2YdBN - LnI9hviID6VMDjTSC4LBucY2J9VnNOERc3wFvdd/YGxTLWjPLfT3XYNkNA09Zdv2+lh70u9qCdvyRJ0T - fPLnA584ifwEp/IzY7D4DI4WyYUSKI3tzU98YHWEIBdnGGGThvXm8uvk7NPZ+cfRtiMBWZLPWUGowBwO - NE4p3Q4bA33fd2vKPLELGs7b5NPN9Pa63XeieBKE/q2Pwl7SreVwsLE79JeSBCCN2pnJkAVSgTJ3amOW - 72rxVyLGH4/UE56FmC0HxPMQXuHrCc9CS56O8CyyTivq1TSMZfpjcnv1qVmFQ1D1EOAipnUPAS79IDGt - Hsi6jgOMtLQ/MoBJksrCkbFM3+5uF03GUJbWuhxsJGaDxcFGWtKZGOrTlamsKS8vowI8xqaskm253ud7 - yY1iKOA4tMJgYqgvyfUc15qp7WjLni5lksnkuawoVoOybWuSZe3R5AvpENsjV2fLgmJpAMuxzAqaowVs - h/pLRnI0AOAgHvficoBxl9Jtu9QzrZZL1rX1nGtcixVNpQDX8UhYn3MAXEcuWD/siPk+TqofKNe23WU0 - kQIsR7N2laBovu8bKAesmAxgIjZOPWS7CMuAbu09Htp/U2ugA2J7aE2312Kvyn2hq+vn5G9RlTrBJEnn - 0ZZd3TG0uq0FbEf2RBFkTy5NTecDYnv2lNy23sRU/xbFY1qsxDrZZnmuH4SnTZVZZVs1PqpfmykXgn6M - zo7/a5/mrO6OQ9rWF0qaqG9bNPEu9O6/TVVuVbeoqB/KraheSSqLtKwPK0pRUd+26cOb1jovREJqHDzW - MddJtVm9Pz/72H3h9Pz9R5IeEgzEOHv34SIqhhYMxHj/7rezqBhaMBDjw7vf49JKCwZifDz98CEqhhYM - xLg4/T0urbTAi7H/SL3w/Uf/Som17AGxPKp3RGsvWsBykB483rrPHG/1aEO1Y8QxVQ+5rkI8pPrVTprs - QLm2kjTsaQHPURAvRgGuY1c+n9EkmvAs9FrSoGDbJlUtlX6CwdMauOsnFnBo1Kr+pjtKNIsmLEsuaDdJ - 833HQB51HhDbQzrr+QgAjlOy5NSybNNKPqqeCmldmI05PvmT2hs+MrapXBNnKzoCsiS/9tn4PQBczjPS - enAdAVnOmv4U3dVykJEpDPtYXWBYgMcg1hMe65mbhx2SeskdhdmSZa5fKVnzrAcatZdrrrkESj65nukh - xHXKkp1iNtZ9abGIOUKMeLf7nKhTBGThDb582HMTOxcHxPPIXxVRowjIUtM1frmT+yVVs19CFlaROHKe - kVFd+bXULqP1JlrAdtDKpVsmVZGi/pIOsTy0x0zu06WiUMlD4fX3fQP1Dugh26VPxKZ1YQ4I6KEmsMX5 - Rsph3yZjmWiDGXcks0t1i6M7f8m+0HsvkdpDgLbt3Pm9wEweabfNw/d9A2WRb4/YHin26zKpUtIaCYPC - bPr/PAies2UtM/ECvStjXVLgWto/04anFmcbqT2jyu8VVeQeUQX0hqRY7StBrEB7yHHVxOc9HeFZGNMv - Jub5aHNlEpgrk/S5MgnNldF6N27Phtir8Xo0tN6M25PRvRFqGnSI5anLxDlQnGD0YdDdnYLJEHeka2V1 - my3OMu5pkwt7d2ZhT3uQuXefZO5pRWHvloWnNN8LYjt+ZCwTcWrNmVc7fmWzL1Z1VhbJI6EGAmnI/lOs - VulPurflcKNeKVNWS664wwN+0rw6BAfc8tdeCMKrEggPRZAi39D6Xz5qeL9/Tr5NvnXbkY1WWpRvIz0K - NRjf9FCVz1STZmBTe4ofx9eSvpXSO+gR36Nfma2eyInWYbZvK7aUp/tHwrbIuiJaWsKz5Ku0Jmo0AngI - K0N6xPMU9J9VQL+ryEVB9eTmm/1Xnz41U9mUKX6TgU3Jsixzjq4BESfpGG+fDFmT56x+1Juf8vVHBRKn - XNXksxJQARYjW7frMGrCnhS4AYmy52fEPpQT+zfIiv1QXpAmSCzId+VqNEO/a1rKt8lduhJUWQP5rv3p - R6pJIaCnO8Ez2VXqo5fxUzkBBRgnFwxzDv32M3LZVAjoif7tvgKI8/6M7H1/BnoYaaghwEW/v/fQfa3+ - yLgmDQGuC7LoArJEZ+rFiDxdybNkSf/lLQb46s17lrDjQOMFwwakqB7xkWvUBrJdxNOxDcT2UDaSOHzf - MWTEl6EtyHXJVVqtk9Vjlq9pPgO0neo/svF7DvUEZKEcmGFTjo2yM+0RABxtO64n58bvuwvCtrtZYKfK - b0LoMLucbaQM3Q/f9w0JuQ7qKdtG/GHe7yGO/gzE9lAmjA7fNw3zbiAgKj0/txbVeJmHQt6s7k6weEwl - ZT4cNwBRdD9an2lJ6of7rG3We4KmWSG79wJeKRUURLv23Su1e2xSto1WC8+9WnjevvBZvBJHpjaHGxOR - iy1ht1iMhyPoEhgbxXUAkTgpA6cKfczugIiT+/sHf3eSbXd5tsroQ2rcgUWiDXddErHu+do94iXfvEfI - d+WprEldbguDfLSxskn5tnKnnwYQV6aC8ICbdVP4hqEovMmhIdNQVF4RhBx+JNIMxBEBPfwBG6oA4+SC - Yc4F4DojJ6ozA3H8Y/RvD89AdF+izEAcEdDDSEN3BmJOfX3GQECPfv9RL/1h+A4o6GX8Vndmo/szuZqF - atiYmQ3MAEShzmxYGOAr6ixXw5lKkjsJBgp4yTMmNgcaLxg2J6doo8a5N2qc65dXDgvjjr0M8UAbJmEO - L1Kz1ZAz7CEGghShOLyf4wtCMdQQi+9XsO0mjbzn7sh73u5+qV8JpliOkO1ql0+2r73m2d8qfykvZuAG - KMq+XjHtB9KxCvGzTWLS4x8HtJ3yZ7ajqPT3HUM9/un/4fuugfIUuycMy2S2mH6eXl0uJvd3N9Or6YR2 - +h3GhyMQaiqQDtsJqxYQ3PB/u7wib7pkQYCLlMAmBLgoP9ZgHBNpZ7+ecCyU3fyOgOOYUbZj7wnHQtsH - 0EAMz93t5+TPy5vvE1IaW5Rja3aFEpKW/y6IOPOy2+GeJT7Sjr2tVPOM0IeyMcM3u0mup/NFcn9HPmMT - YnEzoRB6JG6lFAIfNb0/7hd3yafvnz9PZuobdzfEpADxoJ906RCN2dM8H3/UMYBiXtIcr0diVn4yh1K4 - eWqimlae+UBjdkoP0AUxJ7s4BEpCs/GdXt7DTgnTMBhF1mmdrZrc1uONdCMig/pC7Bpo+ypDrGf+9n0x - +Yv8mBpgETNpaOiCiFNvGUjaehymQ3bak3IYR/z7Iu76DT4cgf8bTIEXQ3VWf6heBvWBPQSjbkapMVHU - u286WslS/zzJDGA5vEiLL7PJ5fX0Olntq4rykAjGcX9zjEl3KDU3iOkIRyr2W1Flq5hAnSIcZ1fqiY4q - Jk6n8OKslqvTsws9+Vm97qj5YsOYWxQR7g723Zul/viUa3dwzH8R5x+8/ig76n5M1f+Ss3dU7YHzjW1r - pvuI1AN8cIMfpa4i0sSCB9z6n4QnIbjCi7PJdjI5vfiYnCW7itopsWHfXVY/1c1Wi1Wt/3slkm26fkqe - s50oi+ZDvdOxfuGGMnXLcPtXRu/Igz345uhwXgEzUc/7sNrqrEvJnYsexJy8mtOGB9ys0gopsDi8O86G - B9wxvyF8x3VfYnW8LBYzNyPCn+KV5z7QmF01zuM3aAVQzEuZV3dB36mPc3tt+7/t8c3cXlbAFIzancP8 - FmFdVTBue6HxQS0PGJFX7T1AZ+PZnx0PtOepjzjob5qGbuvVrCwYIRwDGKVJPcopPBCLmvX6zogsdhVg - nPqxOfFUfZcwrQ/jvv8x1eu06aPDHvScer1rKrdEYUf5trZrSe6RHjnP2FSr8lVSdicBUN/bHNq6ydZq - mJmlebLcUxbzBxxepDxbVmn1ysk3E/W8W84c8Bae/W3/zLlEg/StYkvYM8GCPJeunXg1p0H61v024cyG - HDnPWMaM98rweK8sVtSKUSOeZ1fmr6fv353z+lIOjdsZpclicfOe9pARpH17JRKpqopl+cK6dAf3/NWa - UYe1EOLSO7PV2S4XF5RzXwMKP47gVDIdBdg27UEIarCS6ODNBsKkl0uGRHjMrFhxoyjU83YbMvErTl8w - IkbWLt+JDtV5sIh7yY2hScBat69JR/SxQQcY6W3GL5IwfpFvN36RlPGLfKPxixw9fpHs8YsMjF+aI63X - MVdv0KA9svcvx/T+ZVzvXw71/nmdYKz/2/29me2TQjC1Rxz1Z5skfUqzPF3mghnDVHhx6lyevk8ef643 - enNo/XX1PUFNfMQCRmPM9x4ww7eYJdezT3/QTn2yKcBGmp81IcB1OGeF7DuAgJPUTpoQ4KIspjAYwKTf - eSXcATZm+B7TKz2GbecvVZl9GT8P6qOotygfn5lejaJeKaV4zxQ3bNicfHiJkSu8919P5ocJ79FXbDK2 - SayW76kDNpfzjEwh4luLM/2glCV1WM/8PsL8PmAu6PlzYGxTwby+Ar023dYSJvoNBPQk+2L1KCjHgIKw - 7y5Vh3eXVllNvtSeNKxfSLtcd1+3+OZKCYLm+74h2e2XpAxwONtYbnd71T0n+noKs+lZzkdCnkIw6qad - ZAnClpvS8ndft/jjqWq0ZDQx2KdKYboVtagkYStnVODEqN8lDySnBnwH9Te3iO/ZUS07wPGL/IsUAniq - 7Inzww4cYCTftCbm+35RTb9chz607bffT38nnb8HoJb3cNRRX+4IZh+23IQ+a/ttmyaeU2Aglqd9SYD1 - +1zU8kr6vSShe0nS7wMJ3QfNsL15+5Vm6iDblf1NqV/11y2etnj5CJiOJtUl5YRVkzFM09nkanE3+zFf - aIDWdAAsbh4/2PNJ3Eq5iXzU9M7vby5/LCZ/LYhpYHOwkfLbTQq2kX6zhVm+7sWY5Pby24T6mz0WN5N+ - u0PiVloauCjoZSYB+utZPxz5zbyfi/3SZo53R1laAcKGe36ZzKfE2sNgfJNu46kmzfimrhWmyjrM91Gy - okd8T9N6Uk0N5LskI7Wkl1qk7kT3fdvQDsz0xgNpva9Iv85Bbe+6jFH7tGfXnxCVGvE8T6LKNq9EUws5 - LtXkX38hiRrCtlDvR/9eZA0FHQ4x8gaDqMGNQhoOHgnAQv7lXi/28Ncd2bODLL/ov8vuDR//Sh0WuiDk - JA4MHQ4w/iK7fnkW6oNKBwN95CWWEGubI4abII3YVe4xbmkAR/z7ZZ6t2PojbduJ7a7X5rIHugALmnmp - 6sGgm5WiLmubJaNuk2DdJhm1kgRrJcm7UyV2p1Kbdb9NJw31u+/bBuJg/0jYFnrHAuhVMCYNTKh3Ta54 - c+0uhxub16K42ga23IzxiU3BtpJ4IifEQmbK6MemMFtS8XxJhRol0wj+YuIozQNh5wtl5wYPhJyEVsiC - IBdpBOhgkE+ySo1ESk1dcsv2gXStxHGWBQEuWpXoYK6PfmHQVem/tYfTFHqxdbMcNRfpT7N957yvybP7 - V/e3oEb82ytpnGT30zz54/OuOZwxUT2qx/HnP/ukZy0yWe/Ozj7wzA6N2M8/xtiPNGj/O8r+N2af3X2/ - TwivYJgMYCJ0IkwGMNEaZQMCXO0gvp0fKCuy1cYxf1kRTi0AUNjbbnC4ydMHjrqnEfuq3KQrZpocYcy9 - r56ELoE8+YEO2imz1QiO+NfigVMCexTxsosJWkra25pwcIpPAlY9F7F8jUlmz4BE4ZcTiwbsTYqRJrAB - FPDKqPtSDtyX+nN+ZWXRiL3ZAUa/mKhaYKkP2FXdgy0rEmiyon6d/Ojm2WljNwdEnKRRps15RpXhmSpK - 7ZZjYlWN3+oSFfgxSO1jR3gWYtt4QDwPZxofQINeTrZ7PBBBN8lVSU7OHoSdjPk6BEf85Dk7mIbszX1I - vZc9FjSLYtVUV5JhPrKwmTax55OYlTwRj+CeP5NJuUt/7am34JHzjCo/zwivZ9qUZztMmbOabliAxuDf - LsHnBt13SNMqBwKysHsyIA9GIA/NbNBzlqv6jJ6qHQXadEozdBrzfO1DBHaSujjipz+WQXDMzy69gecz - h2+ozxg39QGDfSo/OD6FeT5uH9ZjQTO3JZLBlkhGtEQy2BJJdkskAy1R0xdndFKOHGjkl1qHhu3cDooN - D7iTdKM/VHmtBlpZkZJmlMf5vCugPXKzIMv1bbL4cnfdbliUiXyd1K87SgUI8laEdklduqY0J0cGMDXv - glJHDS4KeUnzhkcGMhFOcLAgwLVe5mSVYiDTnv773PEafRWpBQGuZl4v5vYJaUbHI07YDKmAuJmeVKjJ - MVoM8skk1Tt16E1panpps3HYXxZtp4YjP7CAebunl2jFACZajxpYL3z8a9M11LM/ZN+RBKzN34ndJodE - ravlkmlVJGqldckcErDKt7m75di7W77d3S0pd3fb09vuKiGlWL9JbFyHxK9LfnXg8FaEbmCTrc8Kwuks - Hgg6Za0+WzOcLWg5m5Ng91leZ13dQylnPmy7df810c9MKc4jBLrOPzJc5x8h1/sLxnUpCHKdn53SXQqy - XM3+i6pAtdnVPA1+2a4T+Zjq/5TyeU+IMSwLxVY/8/B1/Z9xsQGZEfv67Pz89Hfdg9+l2fiHHTaG+g5T - 8ePfokYFfgzS2hCD8U3EtRMWZdqm95ezxQ/yi1seiDjHv7nkYIiP0hdxOMN4+8f0lvh7e8Tz6EqtXZxC - nM+DcdA/i7HPcHdzTtihRhbFg/pIEiNACi8OJd+OhGepxINqkvRp93netNy5qKlZCDq8SDIuT+VQnsqY - PJVYns5myfzyz0kyX1wuiOXbR22v3iRPVFVZ0ea7PDJk3fC1G9vbzkA0H1OcBgb55KsqOFuu1qRte/sz - aEfmuhxuTAquMylsa3NGQvuRpDhNzjHuixX753uw7W6eyVGz6gghriTXf+IIGzJkJd9YAO77C/HSf6vZ - 9pkawjfYUdQf2Vnoso5ZtyyfpnecMueygFn/B9dssIB5dnl7zVabMOBu9p0q2XYbt/3N4cjkW6anMBv5 - pnHQoJd820A8ECFPZc1MjB4NennJ4vDDEXgJBEmcWOVOD9m2afWTZO8xx1fpZWFNSFKxNjncmKyWXKlC - A97Nju3d7BzvnlPi9mBZq0Qqy4JdMQO469+WT6I5ZlPQxD0HGrvNarliE3f9si4r1iUboO2UKScNesqx - HRt06i1rk76VepMeGMP0531yObm8bs4bTwnHbHog4iSelgqxiJk0DnJBxKk7RoSVMT6KeCk72XpgwNm+ - 7LPOKrGinLMz5EEiUkb7DocYy53gXbQGA87kIa0fCWvrER6JIAXhPUQXDDgTuUrrmnnZpgCJUacPpNcd - ARYxU05l8EDAqZdx0PZiA1DAq9/bVM1J9cip6UwYcXNT2GABc/syHzM9TNh2f9KvYC7Kr4TlPRZl266m - 918msyZTm+N+aS8TYgI0xirbEW9wD8bd9DbLp3E7ZX2Lj+Leusq5XoWi3m5PZEpPExOgMWir+AAWNxN7 - CQ6KepvlK7sdrUuHK9A41J6Dg+LeJ0aFAvFoBF4dDgrQGNtyzc1djaJeYk/HJnFrtuZaszVq1QcLcItI - w6JmGV/G5Zgyrr8UUwMc+WCE6PJoS4Kx9Jbb/ArTMIBRotrXgbaVmw94+sfUNOFaJipHB3KSWbOgtQrv - 3vfve3q3B+rrNH/7nBW0cYyBoT7CTn0+CVmn1AbwSGE21iV2IOT8Tjpf0OVs47VYqRL0KZXi4weK0eRA - o77rGUKNQT5y2TEwyEfN5Z6CbPQcMTnIuL4h1zMW6Dl1j5iTiEcONxLLt4OCXkb2HDDUx7tM8D7sPmNl - ew86zuxBSNqPbgjIQs/oHkN9f919ZioViVqpuWKRkJVcdI4UZmNdIlxumo/mlNV7FoXZmPl9RDEvLy0P - JGZl3DYOC5m5Vtz4J21tpMPhRmZuGTDu5uVYz+JmbvqatG2f3F7dXU9YsyYOinqJ42qbdKwFq19jYJCP - XBYMDPJR87+nIBs9z00OMjL6NRboOVn9GpPDjcR630FBLyN74H6N8QHvMsH2qfuMle1Yv+bL/ddJ+2SA - +rjXJjFrxnRmkJHzVNoCESdjht9lEbN42ZVVzRK3KOKl1sgWiDh/rjcspeIwo9jyjGKLGLlP7EABEoPY - KpkcYqQ+17ZAxEl96myBqLPe75J0Xz8mlVhlu0wUNTOGLxqOKUWxps1m4Zax0dqlDvo9HtY+qwx38Mre - ItnHpXh0Yo9I5/+fkpiRutQVCRYIOL9ef25P/N7SqyGDRcwZTwq2mV8n35rdTXJGFWSwiJlzpQ2G+Myd - iblX7DiwSP0OIexAlgKM84PdtzBYzExcOWCBiJPVrwB2ETQ/op4FD8KIm/o83AIRJ6fX0nGIUa9ZZSk1 - iDg5vRR/HzTzE87uQQiPRaDvIATjiJ9Vyx9A2/ntOmLtkgeD7ubulhxxR+JWWn3zLbC+9vAZsa4xMNRH - HBnbJGytBLGesUDQuVb9iqrk/PiOBK3UevYbtlb5G29F8TdsPXH3Aa1bc4RgF7H2MzDQR6z5viGrjru/ - k9fLmBxoZK1fcVnYzKuH0BqItD2ZjXk+dk0ZqCU5qQinnn6Jut1XjaG0Yc9NXMvREp6FkXJgmjHy1M/P - +0+TRDZzhhRVTzm2r1fzizPV1v4g2Y6Ua5v8OGs+pNkOlG9rpwfX69N2WJYVm5KqBhRIHOq6XAtEnGta - e29yiJHaPlkg4mz3qSZ2/nw6ZK9kmpSp2CV5uhQ5P47twSM2X9w+bE6JDSbmGIjUXFJkpM4xEImxYhFz - DEWSMpFpXhMH4SFPIOLxRN+YZDQlSKx2foe4aNCnETuxB2RyuJE4l+OgiFe+0V0pR9+V6ptdJcytaSzD - YBRd5iLDaAUeJ1k391KVbh9EQTuyZNA0NuqvN4z7ayiyWLVf1lOP7JCmZEQsfWHHLfaig1q2QHTGDDLE - ByLoW0aV4uiS43jGRdztl+Jl9xYxW9NA1Jh2WI5qh+UbtMNyVDss36AdlqPaYWm0n11qR/4yy0SI+gbZ - 5+vGx4/phOC6EfHfKvBwxOjejxzu/aRSEhdQGhjqS67nl0ynRnFvu5k7V93SuH3Gv+oZeNXLVApOR63j - ICOnWUDaAMqu7wYDmzhnfMA45NezyDEBbB6IsBb0+RODw43kuV4PBt36gDKGVWOoj3upRxY3Ny/FCdoC - BogHInQvKJPNHYcbeclhwoCbNVODzNKQjhE3IcSVXH9h6RSHGhk16gHEnMw2wGAx84x7tTPsak+ZaXqK - pukpN01P8TQ9jUjT02CannLT9DSUpnUu9X2mFzLTTi4IWuBoSZU+c5+1Y45QJNYzd0QBxGF0RsB+CP3s - PI8ErG1nnKxsMdTHq8gNFjBvM9XvKx5iOiW+AojDmTuE5w31xF9sWQYcoUj8suwrgDiHyRuy/QAGnLwy - Y9GQvdlpsPkWvbyYMO5uc4Yrb2nc3mQHV97AgFtyWzWJt2oyolWTwVZNcls1ibdq8k1aNTmyVWtOPCE+ - d7ZAyMmZRUDmEJoBNev+O5Kg9W/GL/ae2Td/ZqUeknLE0+xsDPA9kV+0NDDUx8sPg8XNlVjpVzy48g4f - 9Ef9AtNhR2K9MYy8K8x5Sxh+P/jwV+KiPQPzffQX2bB3jJlv7qLv7PLe1sXe0+3/Tkw9C4Sc9BTE3/fV - Ry20O+ElaZ6lpO6Ey/rmNXn/hJ5ybHrn31TI5PTsIlktV/r8oKaVIskxychYSbbdqb5HRt0fdpRw+Br0 - WU1v8Is7TSjeapss872oy5L2WjBuGRstuXibeMnFQMQteZdVRBGKU1fJ4zY9pDo/mO0JRHxYbdlRFBs2 - q6FUsW62Eo2J0VsGosmIm6zjByKou+D0LCpGYxgR5X10lPdYlN/P+LnesohZ1xPRNa0rGRkruqYNCUPX - 8AZ3LOAJROTmXceGzZF3rGcZiCYjMit8xx6+wb9jLcOIKO+jo0B37OoxVf87e5fsyvz19P27c3IUzwBE - WasrEWvxPu72BS1jo0XdwING4Cpe4pP2ZTBtj/0omvuIIb66YvnqCvYJwnkoNgb7yFUU2p9oPyg3rOtT - GOBTTRgnP1oM8THyo8VgHyc/Wgz2cfIDbunbDzj50WK+r2t3qb4OQ3z0/Ogw2MfIjw6DfYz8QFrv9gNG - fnSY7Vvm6U9xtiT2Y3rKtjFeMQXfLdWVO7GEdIjvIeZkhwAe2pL9DgE97xmi97CJk0wHDjFyEqzjQCPz - Ev0r1BtOFPucNJF3YGyTfn7dzkotX4t0S8pYlw2YaU/AHdT3tnNevCs22YCZfsUGinvL5b+4XoXa3sdU - NtXZY1qtn9OKlBIu65h3PwW3Q+OyiJnRFLgsYI7q1sIGIEr7Rgp5zOuygPmlPZ08JoCvsONs00r9Oe+K - VZLmD2WV1Y+knMAccCTm4gcAR/ysJQ8+7djXpO3E1ddd/pzGn3t8M5ojShrGNu3ULxVR+Q0boCjMvPZg - 0M3KZ5e1zdXqLPnwjtow95RvY6gAzweawyl71HLjl5lmHmHTbATa7SG2qvSLDfvNJnuhqlGRF/Ps7ANR - rgjfQqs2oVqye/LzRikQUnlx319Q00ARnuWcNvPXEpAloadmR9k2PSmlZ6ia1wK2KekmcVnY3NVPetlA - teboLQEco/3s8E253+kNSAUrGqLC4jaHujLedYMNRpS/FpPb68l1s8nT9/nlHxPaenkYD/oJSwYgOOim - rN0E6d7+eXo/J72gfgQAR0LYQseCfNc+Fwll5ONyjvHXXlSvfavenMe7lyQ5rHDiNMcRr8p9QXiS7IGO - U4rqKVvpF2HW2SqtyypJN+pbySodPzgeFA3GXIqNPhb5DYIaJifqk6gk4bxak+lNf0xuJ7PLm+T28ttk - TrrNfRKzjr+5XQ4zEm5pD4SdlLfwXA4xEvaXcTnEyM2eQO60L86U+qDeW0IFElCE4jyl+T4iRoMjfl4h - Q8sYt4gFSliz/JrlbEjEKo+JX3Dzz1aE4vDzTwbyb/7902I24RVvk8XN9MLRk7iVUUQMtPd++Xo9+hQi - /V2b1Fvep8WaIugQz1NX6aomihrGMH27vBptUN+1Sc4Ony6HGcfXxi4HGQk7e1oQ4iIscXU5wEi5kSwI - cOn55vH7HjgY4KMs/7YgwEW4AU0GMJH2s7Qpx0ZaTt0TjmVKTaWpn0LEpdMm45hoC6YNxPFQ3v04AoZj - Np/rV/LT8XfykXAsoqBaGsKxHLbZpkxAeqDj5E9hI7jj506cgrDrLvPX9+pmVaOMmuY1QNC53ecMoaJ6 - 23Q+/66+mlxP54vk/m56uyDVkwge9I+/h0E46CbUfTDd27/++DSZ0W4sA3E9pFvLQECP7mDobmmu/llX - hEY35HAjcW5jnwxZI39GUOXGjXjGhgrQGORqBOPdCOxnRwiO+JnXj9eD3eftJ5uq3FJfBUYFfYxv16Mf - B6ivWhyte3IEbAelc3L4vm1YVKqnvimrLUVzhGwXrXPSE6blfDx+bnHU9Dz30/OcmJ7nXnqec9LzHE7P - c3J6nvvpOVl8ubumvE7bE55lX9A9DdObmgmIq7vb+WJ2qRq/ebJ6FOMPvITpgJ3SqwDhgHt8QQHQgJfQ - m4BYw6w++UxLgiPhWppdg8WqJkxyeyDorCvCEzOXc415Of5QvZ6ALMkyK+kmTbk2SnYeAMMxWcyvLu8n - yfz+qxqEkTLTR1EvoSy7IOqk/HCPhK3TZPnxg+7qEh77YXwoQrtbBD9Cy2MRuJk4DeThtLkrVFeF0H/C - eCwCr5BM0TIy5RaRaaiEyMh0kIPpQNnYwycxK22TCog1zHeL6dVEfZVW1iwKshFKgMFAJkrOm1Dvuvv0 - 38lqKc8Ia4ENxPHQJqUNxPFsaY6ty5OOf+oJ27Km/ZK1+yvUf6x1Uc3WetGApLgcFPUuX2PUHW3bm6eS - qvObUqRHyHOpjut6fGfXgmxXTjqQvCccS0Et6C1hW9QfzlbLJUXTIb4nL6iavPAthBX3BuJ7JPlqpHM1 - SktN4g7xPfVLTfUoxPZIco5LIMeVlqrpEN9DzKsOMTz3k1v9Jb0vSprn/YokmazKYvy9FtYA8WTz0J4e - oON8o14BVK6ovpYCbLSHrA6G+AhtgI3BvorUk/BJwKryKnsgGxsKsO32qmFoTlcmK3vU93J+Nfx79fzh - y1q1XzXddyB9q250svT9GWGeH0AB77bOtuRf3lKYTd2x/+IZNYla19lmw9Rq1Pc+pvLx/RlV2VK+rUvi - 5J4qPIKAUz8abjbVLsnWHgW8Ms2L/ZbsbDHYt3tMOT6FQT7WDdRhkE/u0pWg+xoM8r0wLxC7v/PHZC1y - UZOv8QjCzrJpOasHjvbAgmZOhdlhoC9TTVxVM4wtCDoJg0+bgm37rRrkivHb10IsaK5EXWXiiZOeBzTo - pTxsQ3DA38yD7rO8zopuXTs9ZQCHH2nL6oVtkV5Y+3fSmigABbxiu6Z3SlrKtxUls+N0BH3nrpTZS1KX - SU2u+Q3U91aClUEd5vukWOlDe/jdUU+AxuAVLQsG3D9VlSx2pAWLEIuYOa3EEQw4k2zD1io2ZN6N3w0F - hGE3/W5rKdCmp50YOo3BPk65/YmV1p/M9vEIwk6ZSNKLcxALmhktb0thNtJGGwAKe+ld4JYCbbuSUx4V - hdmawkBYTQrTsH0vHzlahYE+wkpem8JszcFYm32x4mmPOOx/zDas69UcbCxZ96bGQB/ppQ+XA41/i6pk - CDUG+OpqlapWcEsv8UcStHLq9IYCbXqoztBpDPTlq7Rm+DSG+BgdhBYDfQU/U4pQrhS8bCmwfCkIh0g6 - mO/TEzwP5Hq8pQDbVvdym+4uWdmjgLfMy2dB7gV1mO974k52P+Gz3cePVJ+hXe/Klh8NfpS/WV3uv92+ - 9uLLZEZ+QdOmIBthUGgwkInSBTIhw7UTBfwAZLQYNeBR2i2/2CE6HPe3Oy2w/R3u+4mvZjsY6iN1En20 - 995PviWX89vT5kX6sUYLQlyUJWweCDifVQkRZGFDYTbWJR5J2/rX+bvfk+nt5ztyQtpkyEq9Xp+27cvX - WkiW2SZtq/rP5lnjMh2/stblHGOZPKpQ49spC7Jd+rGT3vnkanqvarcmdShWALf91Nz387xJ1esvtDPJ - PBByzi/v2xcIvo6feIVp2J7cf/9EON4LQGEvNykOJGCdXEUkhQmDbm5CHEnAev/1av4b2dhQiO2CZbvA - bOrr0z+b7XKoNxXmgCLxEhZPVX4pCJaBWdS9Nhu41/TnzWtBXPkBht3cVJ6F7mPdGJGNGkJcyeX3v1g+ - DWLOq9kNz6lAzDmb/JPnVCDgJLbUcBt9+Cu/nTFhzB11D3gGPAq3vNo47o9JokAbpD+PaodcARojJoFC - bZL+nNcuHcmA9YJtvQhZI9spxINF5Cd8ONXjSs1gmZlF37uzEfduVDvmCvAYMbkwG6ofWO3aAQw4We2b - CYfcnHbOhENuTntnwrabPOwHRvztkJ3T1NkkaOXeKACO+BnF12URMztB4Fat/ZDbpPk0bGcnB9KStR+S - mzEDw3wXPN8F6otJWEcwIkZCWLkflKCx+E0xKgFjMQtMoLTEZEQwD2Zx9clsqD7hNrk+jdjZqT0L1lbU - ZranMBu1gbVJ1EpsWm0StRIbVZsMWZPbyf/wzZqG7MRBKjKnfvxzRNuNj1ONz+PuuYGRqvUl9t0RGqta - 34hKqFC7HjNchQ14lKhkCrbzrCGrg4a8F3zvRdAbm/Aj2n/ga7w+ACIKxoztC4walxtfjShgA6UrNqMG - 82gWX1/NxtRXcX2F8Pjc+k5UbswGa0Ve3wEeo9uf8foQ+Cjd+ZzVl8DH6c7nrD7FwEjd+pzXt3ANRhR1 - e5+eJfefJnrdxWizRXk22qYHFuS5KIt+DMTz6KfMeoO/tFgnK1GNX5aC8V6EZts6orVhPFO7+Qfl0BYP - dJzJtz8+n5JkDWFbzlWGf73+fJZQtqH2wIAzmX+5PGWLG9q175biTG8PpF+PJL0JhOCgXxRRfhO3/b8l - y32xzoWud0gF1gIRpy7F2UYfhCF4blOAxKjS5/g4rsSNRa0ifgNqiN+aG5yezAcKsun6l2c8kJiVn6SQ - AYoSF2HIHlcsIIMbhbKjU0+4lvp1J/T7L5RNaHwStTYLHJnehsXMXY0i1jz5Ecf9TyIvd3x/h2N+nRdc - ecuGzZfFehL3E3yPHdEZMpHrKIgPR6A1PT4dthPWOCO46+9aVZq1g1xXV2Bprg5yXYfdk483AWef5BEq - N2676/EbRA2IjJh3N9OrH/SiaWOgj1AQTQh0UYqdRbm2f36/vGH+WgtFvdRfbYCok/zrTdK1snfRRfCg - n5oa6F66wMfkVMH30+0+/3Z5f69J+mUbJGblpLWJol7uxYaulZ62BtlbZ5e310n3jsRYn8k4JvUXkb6S - RC3ieAgzHIfvO4ZmkT7J0RCQpT2aVp8OqndS1od7EzqZAxonHnH7MJNxTOtMpks1JNuU1c9kX8h0I9Qo - bbMRlD2fh01OVPFAyzf1fddQvNFlh0ROzE1GPDfUphxbO+gp1slW1I8lLT0cFjDLV1mL7eHQC/3zktVe - 1s35CMQUGtY58ZutYfTPJoU5Uo5tV47fPeAIuA4p9uuScbOboOOUQtAyTQOeg18GZLAM0M6gNRDDczX6 - 3Az1VYtrLo7QzzUQw2M+fqFsGeKBtvPwrIWqNDnL+L/J6buzD3oTJH1SYJI+vZwRvABt2ZP7+Ty5v5xd - fqP18gAU9Y7veXgg6iT0PHzStuoXSHc/V/JU1TaCcHg8xNrmZTb+ucHh+44h14cPFw/J+PdXHcz2Ncdl - qHpwR7qunoJslDvRhGwXcXxvIK5nk+7zmlrneaRtJc4YGIjt2eTpAynpG8BxEG9T/950jrCiyBw04KUW - Mg923fW7ZFXVCW11DYAC3jVZt4Ys290pXaQg0PWL4/oFuQRZJADLJl3VZUVP+I4DjNmv7Y6s0xDgIlZC - BwYwFWRPAVjoPwz6VTspueW9RwHvL7Lul2dRdz9tDGpjoE9vyqVaLmqVZLO2OZNJuUt/7Uk3wRGyXRGn - +SE44iefhAfTtp3YZfL6STqB6a1qT2E2vTOl4Ckb1Pcy88dBg94kT6sHQb9uQBGOo7ftrOqYMK1hMIqI - jAH9DlY5tsmQlZ0JnsGOstPzY6r3rHv37eqWu8vJfbJ92JDa5IBmKJ4er8SHO1iGojVPKSNjtQ48UlEW - ghtBs7C5HUy8QR6BouGY/JTzLW405pmrIAy6WXcnftpq86ne5Iuk04DnaC6bMSJ0UNjLGMs5KOxtxi36 - jFjaRCBqwKPUZVyMugQjtHnKSXaLBK2cRLdI0BqR5JAAjcFKcB+3/ZI/opWhEa1kjtYkOlqTjBGWBEdY - kjdukNi4gbJu6/B939AMlqgthwUCzip9JusU45r+FjTL305LqYpdTZ926inbtt9RThLuCdtCO+mwJyBL - RIcJFIAxOOXDQUEvsYz0VG+jrIG2Vzzrf9GOzO4Jx0I5NPsIOA7ysdk25dhoB2cbiOU5O/tAUKhvuzQ5 - fY+MZyKm8QHxPOSU6SHbdf6RIjn/6NL0tDkwnomaNh3ieThl0OJw46e8XP2UXG9Le3Z6Xh4hy/X+glLO - 1bddmpyXR8YzEfPygHgectr0kOU6Pz0jSNS3XTqh3SkdAVnIqWxxoJGY2iYG+sipboOek/OL4V/L+KXg - r+TUERbnGVlp5qXX9P7L5fxLQmixjoRhub/8OjlLrhZ/kR4zOhjoI0w/25RnOz4p3MoHotJEPe+uKldC - d9fIWoM0rKRliO4KxPbf1M2rbaq3LWbf54tkcfd1cptc3Uwnt4tmYo0wpsMNwShL8ZAV+ry8fVqMP2dv - UESImZQqNZKtyp704e0uwLKOuJpKrMV2VxOycoQqGFf9PZOPb5H0jmlM1Df5uZ4rHJlQXyF40E+ov2A6 - aNczHLKqIu9IwwJHm87n3yezmHvfNgSjcHPEwIN+XSBjAjR8MAIzz3s6aNcFW2wjArSCETGi60DcFoyu - y+NW1KmeuIsscK5qMG7E3eRb4GiKbf+DW9ItARxjLVblun+Wc0gCTjREhcVVXzMeSUixqsaf5TVsgqOK - l5369lYUdfJ0yglmCYZjqK7bdhkbp5GMifVU7qpNfLRGA8fjFkS8/JnL8jhmk4cjMCtZtHbdSZ333Izt - 6aCdnZUm30f4Pp/Mbu8W0yvasUUOBvrGj3otCHQRssqmettfZ+fnp6P3Amq/7dK6LO3SrKJZDpRn657U - NZVTVzkSzYDBiHL+7vc/3yeTvxZ6k4Z2QYM+iXd0DIQHI+gde2IiWDwYgfBWnE1htiTNs1TynC2Lmrmp - MJgC7aeJ/BkjVzjoX59lDK2iQBulPnEw0PcwvhdgU5iNssGdT4LW7IxjVBRo45YivAS12c/73UcWNJMW - 4Lgcbkw2O65UoZ63O2mv7QxSZgkw3ougbrJTRjE4YJBPv8JWrNNKv0lVi0JPsEm6HrKA0UgnvbocbkyW - ZZlztQ0ccNPLnsV6Zh2uy+ea8u4tgnv+5lZiVJBHzjP2mcq6FV3c8+taj94+dBRo492BBgla2WXNhgNu - euJarGduFzbmmaRqe9BzNgdO1y9EYUeBNk5bdORsY3J588fdLCEcC2xToI3w1qtNgTbqrWlgoE+/ysLw - aQz0ZTXDltWgizC2sinQJnm/VGK/tJl+W/OMCnSdi8Vs+un7YqJq0n1BTESbxc2kXUVBeMCdLF+T2+l1 - VIjOMSLS3af/jo6kHCMi1S91dCTlQCOR6wiTRK30usJCUW/7ZiVhyhXjwxHK5b9UcxoTozWEo+g3DWJi - aB6NkHEvP8OvmlwrmiRqVZXSaUyeHvlwhKg8NQxOlKvJbKE3rqYXeYvErMRsNDjMSM1EE8Sc5N61g7re - 6e1nRnoeKMhGTceWgUzk9Osg1zW7oe8u6ZOYlfp7ew4zkn+3AQJONdZ8l1Tiqfwp1mSvCcPuUz16o845 - eDDs1p9ytJoDjNQ+f8cAprXIhX4xinF5PQp5SZvdOhjk29N/sd/b0H9l3TzIfdO0qaq3pLcmJjtNOOCW - osrSnG1vcczPmwmDeCxCnsqatkAS47EIhbqImAg9j0XQ7/ak9b5iBjjisD+ZTf68+zq55sgPLGLm3NYd - hxs5wyYfD/upgyUfD/tXVVZnK95t5ToCkeijY48O2InziC6LmJtVVRVL3KKIN64iGKwHIquBwVqgv4up - z31gAxKFuF4YYgEzo2sH9uq2ab16JKsaCrBxuodwz5AxmDhQmI34xMwCAWczGoy4BRweixBxEzg8FqEv - xGn+UPKi2I7hSORHaagEjtVVXKTdWzEeicC9r2Xwvqa8Pm1BiIv6sMMCIWfJ6BdrCHDRXl12MMBHe4nZ - wRzf5K/F5HY+vbudU6tai8SsEfPViGNEJGoXDHGgkagjOotEreTRnY2i3uaYG06nEVYE45AnNn086GdM - a0ICNAb3FgjdAdS+gkWiVhmfq3JMrsq4XJVDuSpjc1Viucqbb8TmGm/u7r5+v28mttYZbYxho7B3VVc5 - R6o52EjZp9zlECM1LQ0ONj6m8pGbnAcWNpO3agdhx92s/ZrcLmbTCbm1dFjM/COiwcQkY2JRm0xMMiYW - 9SEvJsFjURtoG8W95DvAYXEzq/EE+HAERkULGvAoGdseuieoTaiN4l4p2JcrRR30RuWmHMxNGZ2bMpib - 09vFZHZ7ecPKUAOG3M3DoaKuXunmIxr0sitP1zAYhVVtuobBKKwK0zVAUagP4w4Q5Do8U+NlrEmDdvpD - OYMDjZw2Amkd2nSmT5m7MOTmtTlYa9MuCSJOklskYuVm/BHFvM3G2uw72jUMRmHd0a4Bi1Izn0FBgqEY - 7B9So0+imq/ofjddrCnMlpT5mmfUJGTlNFpwW8XqeSB9jrIQeVYwbuYOhJz0xwc9hvoIB3P4ZMhKfTLh - wpCb1Yfze2+qtE+u6K+smRxu1G9t1KqWk1z1UQDHaOpm/QeO/wijbvraTYeFzdR7q8cc3/33T/r8XnLe - GRxsJL5waGCo7x1T+A43tlvxcr0tHbKTN+sOKOA4GSuZMySVqeWqx2Cf5JUCiZUCGZVnEs+z2f3dfMIp - ZD0YcNKfMXp0yC7j9DLs1x0a4toHjw7bo67/KAjEoA8vPDpgj0icYMrU1V7yr7qhETv9tjxyjlHvPMB7 - WmCRmJVYuxkcZqTWcCYIOJtFwGldV2TpkQxZOSMeSDAUgzrigQRDMahTMZAAjsFd0Orjg37yMjBYAcRp - jxZhHB2CG4Ao3WQRq8QaLGSmTzP1GOQjTjJ1DGA6Jj0r8ywasLMqPqTOO/T4OLlvsJiZt6LZx2H/aSK2 - aZZz3B0Ke3mF9QAGnNzK1eEHInCqVocPRaB3bXwc8UfUqjaO+PkFPVjOI9bsggYsyr55AkTv2kMCJAZn - /aDDAmZGpwrsT3G6UnAvij4Vd6QwG3UizgRR52bHdG6gdil2ZS3iGI5EX1mLSeBY3Dtbhu5sGXvPyeF7 - TkbcczJ4z5HX7B4gxEVes2uCgJOxLrbHPF/zdhL/7UpIgMcgv+/ksIiZ+Y6kj2N+cv/2yCFGRk+0BxFn - zPuCiCMUSb+qu0r1/kTX1LcZAp5QxPZNydv9dikqfjzTgkdjFyb47TznU153FlIMx6F3aiHFcBzWMt2A - ZyAipzMNGAaiUN/gA3gkQsa7+Ay7YnoP78ghRt1KvsFN7msC8aJvcVfixJpP/6DXvQcIcJGfQhwg2LXl - uLaAi1i6WgTwUEtVx7imxd1s0pw2s8pFWhBbU49G7fSctVDU27Qb5C0EAH4gwmOaFVEhtGAgxr6q9C7n - K+JCfFwTjkd/AAgJBmM010LsZqOWcDRZl5WICdQIwjFUw6QfCRF3UcEkoVinTbmU/DidYCBGXMk+HS7Z - p7ooxv0MxYcjMF68Bw2hKM0j0j19yTMmCcaKzJbhXOnriajK09IE44mqKiNyqOWHI6gh465+jI3TWsLR - Xugr7EHDUBTVaLdrO+NCHTVovKzIuCUhKzI898k9FZNErd054Oya5ciHI8S0knK4lWy+0jUGenvs1c+Y - WJYoFDOqfpGD9Uvz+ojYpPu8jojRGQai8O/2Ix+MEFNvycF6S0bXJHJETaK/QzoHHeODEXb7aldKERGj - MwSj1Nk2JoTGB/2JuorsJTJKKwnHIq9NAvhghO7Y9NUyIsrRgUZ6iwpsuO7SM83M3soBxb2sQVdHota8 - LH+yhtQ9DLqZo2l0JG3socupIkwc93Nb0oGxZvOudN7NYXGu3haAMXg9GKz30jwC5KZGD2PubgUTr0Rb - PBqha5nVddSPkhnFcgQi8dr3cNse0x6G20L9abtZCTf1Oxq181vZoRY2pkUKt0axLdFwK8TY4cgEHWd7 - aB15BrnHUB/9obvDYmbG+nKHRc305zkOi5rp96DDomZ6OXZY0Exd8X2kHNufl4y9dA8Q4CKO2/+E3ozX - f6S2cx3jmiaz6ecfyf3l7PJbu3f0rsyzFW1dBCYZiHWaPJbEjIcVoTj6YUfFKLyYJBSLXkxcOmR/YDWx - sGIoTmR6Yfe89aWseFTNRET+d4JQDEanHuBDEci3oQOH3Lr/yJdresjOWMCMOAYjxd3rR8VgnGwXGSXb - jYiRpHIVHUdLBmM1VWkmZGS0g2YgXmwNI8fUMDK+hpFjahj9JV1m3iDWUTMUj9PlxyRDscjTa6BhTBTG - JFvAMxiR3CGEFU4c9urMwKrM5qNKNEtsGVss+Tjkb34MW2/Svp28Qg9eQ9qcb0wfhfUY6CM3gD3m+Jpn - IJyRpwl6Tj33kv4kDuV6DPStUoZtlYIueutucKCR3Ir3GOgjttYHCHGRW2UThJ16qQEnf1sQdHLfeBx6 - 27H7nNEAWSRopVfJBucaiRuJ+XuIqb8cFzOQG0EXBtwsZ8DFaD5t1PEyV+qjK/QZb7KCb7FSV/j7K/ub - moc+kO4xx6f+a91MmbU716fqX4yDhlALEo2z9MhhXTM1RYC0aOYl0339WKpR8ytnHRZoCEdR1RR1rhM0 - hKMw8hQ0QFGY74KE3wFp54jL+nJTc/LgQCLWT2JDXV1po5CX8Yob/oa28UmyzGpZV1xxh0N+9jL4oTdc - It4tD75X3n7YvbHHvXNsHopQL6W+hDR/oNt7FjLvszXjLtGUb+NMTqFv1jcflCu5o+s05dsSY5slqtNk - AfPhaateBJGklUjJfs8wFIW6rTokGBEjEcVTdBwtGYpF3swdNIyJEv+TDpZAtEOfPyabDAcQibOuDV8X - G7UadmANLOetQvhtwoi3CINvD0a8NRh8WzD2LcHhtwP5bwWG3gbkvgWIv/133GxjLdZNO7eX6YPgyB0F - FqfZDYc+jQzwQATuqVwPwRO59Kf8pAmlCLfbGui18jutoT5rs14pFwXZ2XGQkdUJRvvAUV3UgR5qxK4w - QzvCRO0GM7ATDHcXGHwHGP1yJ7vQbgOldssvtlu83G6baZ90/S+a84g5PqOGIM+8OWzATN7o24UH3ORt - vyGBG4PWxHkrDdQdna3pzzx6DPSRn3n0mONrFss2XcxVldO7xD6O+iPcqJd/yfDVUhdq+GszdmklRbKp - ym2y3G82xLrEo117s2SqnTaniQ3QdZJ3mYJ2mGLtLoXsLMXd3B3f1521TxWyR1U3o8SYDrdIx9o9320W - kZGkJug421NsOW2aRSJWRptmo5A3Yt+v4T2/ovf7GrHXF/f9L/ytr5gzecPn8UpuP13i/XTJ7qfLQD+d - uXsaunNa1P4nA/ueRO3INrAbG3cnNnwXNvIObMDua6yd15Bd1/q7a70ndkRtFPXS2zuHdc1GdpE7zy4c - cpO7zx49ZCd3oEGDF2W3Kyv9JuBxloMYw+OdCKyxEDISOvyZ2pUxONfYLK2iN+wG5xgZK5TAtUmM3Q3B - nQ0Pb/JQX+U0ONzY7d8ga3XrPXD1lsSO9fSes8Ktpzwbb92FBXpOxnx2T2E2xpy2B4fcxHltDw65OXPb - sAGNQp7fdtnenJ5lyfReCWaT+Xys0oIQV3J7xdIpzjAus6RWI5JkqQbG++JZrzGpxVZVuun48/eCknCs - 56osHlT19JBJQkd02AREXeXlUvXYkur0HTmOwQbNpxHm06D5LMJ8FjS/jzC/D5o/RJg/BM3nEebzkPmC - L74IeX/ne38PedMXvjh9CZmXO755uQuaI655GbzmVYR5FTSvM755nQXNEde8Dl6zjLhmGbrml+2WX4Vq - OOw+jXGfDrijLvx06MrjLn3o2s+i7GcD9vdR9vcD9g9R9g8D9vMo+3nYHpXsA6kelegDaR6V5AMpHpXg - A+n9Mcb9Mez+Lcb9W9h9EeO+CLt/j3FDPYjmuBvVbW7fXF9nlVjVhzUo5FghGRC7eQc0LqKvAOLUVbrV - D7/Gn5IMoIC3G3FUot5XBVlt0bhd1un4KRUQDrnLHV9dmr07IU/PLh5WW5k9Jeofyc/RC6AANOhNRLFK - Xk4j9J0BibIWK5ZbcYhRrJZNyGVejn9kixuwKOrzrXxIXj7wQhzxIf9FnP8C8f9cb1hixVnGs/OP3HLo - okEvvRwiBiQKrRxaHGLklkPEgEXhlEMIH/JfxPkvED+tHFqcZUxWddW0T4Qnlg5m+x6fk9VypX9A9bqr - KUqb9K119f7s8Gmbt5KqBxReHFUyGVfeUZ6tK4sMo0H6Vp4RsbW7XLSJQiwGPg3aD0nOsxu0bS9Kfmlz - WcgcWeJQCRCLUepMDjBy0wRPj4hyAvFIBGZZgXgrQlcBPtbpMhcfSVuOwzRuj5IPuVVH//Vp/PMkjIci - dB8lj2VVEJ5vILwVocgS9SVGMbdByEkv6DZoOGVxql/A7B6/JrkoHsZvHwTTjn1dJul6SVK2iOPRHQTK - W9QWBLhIJdaEAFclSMehuBxglOkTXach31Wudd6QFjkAqON9EKq8p3n2t1g3yyvqMhl/bBNu8KLoHXLL - bCVURZeLVV1WxBgeD0TYZCJfJ7ua7j6SgLW7J9oqaFNWzSidsE5iUOTEzGS7BEp/jRTDBB1nJTbN43Jd - GTUzSM1Mw9+iKkkRcA0WTzdrZSF4UTrYccvIsiQHy1L9uhPUo708EHLK9rykilp6XBhyNwtlk1SVgVKV - AVHRA7gGJ8q+XjFrCIvsrUsh9sm2XKvKWK+b1BdQUTZ8wXgjQlZ2c6VSdV6p51LAtG1XfyrKRD6W+7yZ - ahy/mAOmbbveD0ndZXppnk687jL0n9L1mvQ7wiY7qv6QnlI95dv0qmP131Rdh4E+bpIDuOEvklRvq7Bf - JquykDWpNAKsbV6vk+eyGr8vg8nYJinbN3Zqqcp+snytBUkK4JZ/mT2oTsM6SwtdVqjXDNCWfVXuXsnS - HrJca9V15+SUxVlG8bJTdwVB1QKW45Cy1B9pcbZRv620LYv6odyK6jWR2zTPKWaItyI8pPWjqM4Jzo6w - LOriq7R4EOSfboO2U7ZDE3XXkq0O6norkad19iTyV91zIpUggLbs/0pX5TIjCFvAcuRqpMcp3RZnG4WU - Sf2obk2jMMwoalCAxKBml0Na1m2W581iqmVWkIZ8EBswq35Pc6YJW38QODGKTN1yyXO2Hj8qdznbWK7b - k3QY5cNjQTM19yzOM6pqsiky5KrLhz131/97196G/DCoB4vITn2PRyNQ6yWPRc1SrCpRRwUwFV6cXD5m - G30QKTONPB6JEBkg4N/u85hGF1N4cbj9TY8FzZz7+Mh5xv3pR/a1Wqxjbo8qpo66ART2UlsMk4ONulMx - mzHTAnH4kYp3VG/xzrbs8w8vzScU0RFyXbyWweQ846rcLtMPRF0Lwa4LjusCcDFy1uQ8Iz0X4Dxo8pne - YXdR2KufRnGkmvOM5CrzwHgmTpkDy9sL63Z4ge6HUpXponk9WQ8HyuVTVu6lGg2oAqU3C64pJWfQZUcu - mtm0vmWhRHJZy7wrn2mlqgUsR6XnlXjjQBf1vV2fo/kOVWyytlms9yuhkmZFcvYUZtMD212ecrVH3PHL - 7G9G2hqY7et6WmShyQHGQ3o3/yB7LRqy8y4XuFq5SuuaVuoPiO1pHieQr8vEHF/NHjl6rGeWtRqnrhhX - a6OelyMETL+qC939UolcpJQmxAYBJ7Hy7yHXRe+59BDsuuC4LgAXvedicZ6R2o4fGc9ELh0HxjW9sIvH - C1o+GKMleKRkta/k1ANoy77nTvzs8VmfPXcQusdHoM/kyfRnYDa9SV2dJv2DBYrRpw17qZ+mSpnrOnjT - Ps1+3KYr1eakZ+ej348Z0ITjxYcaGeV8/HttuKGPsjrLksv57WnyabpI5gutGKsHUMA7vV1M/pjMyNKO - A4x3n/57crUgC1vM8D2m6n9nzeGar6fv350n5W783qYwHbJLMb6Gg2nDrpeNlc0aslWux0ii0MtFRt+j - GN9HWOtku7rSGyBcT+ZXs+n9Ynp3O9YP046dV+rWoVLXf/jtnqs9kJD17u5mcnlLd7YcYJzcfv82mV0u - JtdkaY8C3j8mt+qzm+n/Tq4X028Tstzh8QjMVLZowD69PGeajyRkpdVFa7QuOn5y+/3mhqzTEOCi1Wtr - rF7rP7haTNh3lwkD7nv198Xlpxt6yTqSISvzoh0eiDCf/PP75PZqklze/iDrTRh0L5jaBWJcfDxlpsSR - hKycCgGpBRY/7hkuBQGu77fTPyezObtOcXgowuKK9eM7DjR+vuBe7hEFvH9O51P+fWDRjv374osCFz9U - pfb5rmukSQEgARbj6+TH9Jpnb1DHu6/L+/bYja/j383wSdv66XI+vUqu7m5Vcl2q+oOUGh5su68ms8X0 - 8/RKtdL3dzfTq+mEZAdwxz+7Sa6n80Vyf0e9cge1vddfdmmVbiVFeGBgU0JYOOhyjnE6U+3d3ewH/eZw - UNc7v7+5/LGY/LWgOY+Y5+sSl6jrKMyW3F7SqjAHdbzzS94tZYEBJznjXTjkHr9NNcT65v0yz1aMhDhw - npF4opVNYTZGkhokaiUnZg/6zvn0D6pNIZ6HUQ0dINs1uWJc1RFyXfc6gqhFJWm6nvOMrJvQ5HAjtby4 - bMBMKzMO6noZN8sRQlz0n47eKf1H1B+N3SeqyZjcXk+udV8n+T6//INUrfu0be+G2OTmwuRw45yrdHoa - 0/n8uyKYraVP2/bbyWJ+dXk/Seb3Xy+vKGabxK1TrnTqOO8WU9Xdm3wm+Q6Q7br/ejUfP0vcE5CFegP1 - FGij3TpHyHf9RvX8Bjg4P+43+Ldd8KtbAA/76Yl4Eah3m8/11MmfTU2iR3VkvY0P+lkp5CuG4zBSyjNA - UVjXj1wx5xq9q9Kjwx/krDtSkO2f3y9veMYD6VjJjTvUsvOadaxNZzXoSGvO68Fh/beI6iRUk7ArkUD9 - wRk0ISOmGXc0OsNHo7OY0egsPBqdRYxGZ8HR6Iw5Gp2ho1HzE04ymGzATE8EA/W8yf18ntxfzi6/zYla - gwSs5LpohozKZ+xR+SwwKp9xR+UzfFT+fT6ZtR1GirCnbJvexZ/i0d/3DcnlzR93M6qnpSDbYjGbfvq+ - mNCNBxKyfv+L7vv+F2DS87ks3QGEnKqlpfsUBLlmN3TV7AY2kXuSFog4ifeYySFG2v1lYICvGZLPiesk - bDJknfO1c8BLnRg4QoCLXqEa2P9r7fyaHMWtKP6eb5K3aXp6Z/cxqWRTW9nKJp6teaWwwTZlDAzC/Wc+ - fSRhGyTdKzgXv3U1nN8BgYSQxRHB2/zzfzBMa2iS7E68CRmm5E686hii4E4cZCTv2x//xiaVTHUEERw6 - vWkI0re/4a2M1hAkyTWgy19Q9k65H4f1QtPrj1b7bPl6l5TWJTfn9tIXdg3xNsvNAusmbuQ2pQ/xiZMm - ripLbebLuVg+Xd0RuazhBIFAPUc0sopd+q9frx8R6+NfSvNkNC/fVhKeltG8fVEVZ/PNs4R6F8fYw+K3 - SGxIjBFzOl8quYUWx9jDdzJy/KCPOajvnRyvxTG2mZK87grcCLSL+XI1bbvCVF2Jx1RPOwivLXtVzXTS - baYKIdRqY+R+d5SjtZhnryjmiTzCt2+6605hygic6lL1ZvXCXZMX5tumKutMcgp6c3KYwE+V57ayi3Gm - 7/rh0nR5WWc9euUZCue2su1jKHE3YS0nGZzToWsu7RCReOlehYXoQeJe6hFeas7Lpkz0MotBy5JVmpkW - bm8auQ+hg8OIODX1mrKaADgPG9dnE7JkFqM+7oBkKHD6uIO5JfTdvu7CkKior0qL75esWmF3JTgu2d78 - dc11ymrYg9RTDsP3ozh50FFEXXA3Wxw7Ebts9LVgqnFI2/JQX2y7aBtIgOcpGerw5BJhB6nDXfGQiz7Z - bu9kb//5268IcyJzeMPDBns5umsIEnq/T1QETfTYjj6rh411cYCBWkORdDttonDTc6ZOOHOqJuhAiO5U - Q5Dg5mIqo3iXLQ67bAnS8JWmrkkw765kqKL7hux3mR7StEqavFwUzzJmneCWiYc4XnZZeX2+tp+RtsnL - T+n7Ob9+WZoq9XYBPOdhMe/nnz/fdjd/rvMmYAu9X54Su3uad9m+//TlIcfgQ8ljub43eccu8KdBSz3N - scrPPQ50jkE4UMGOT9w7TPowhi4JQA3FM2z4pZxDOD6tGWgF+0p3jUuyvWHTuphVHRCcIySY9rF6qU35 - d4VSRQ7DAwLhYoYuJIPWLIDxgFtWXxrlouNapH7OAbsPaUDcA6+lHGLGx45VrbKxhCUu6wuOHVm7vYmC - /a2pjOT1t4ZjfK4rAZ/CEH6C/pMrdJnD9ReUiiN0mCbbq7FdaNuDhqsyqXccrlcaezkaRRTLvuigSx4w - coovemEKtCwZj55jAZRHWb9+WuXhAUgPBa2AEggpppv3iqNdPeWAvbCOIooF/4Lm6CgiXK0dHUmEXi9H - EcUSNGWekqGuueRMFiOzg7mx5a0Gi3J9h7FTle2vw5uIka91ycOY6fpKHuNEHB9SlMuI06MwkxLyJn0t - unL/IezO8gzfSZWHOn0r+6N5ou2GpaZOdfNWp1mt3opOYLwIOT2O4bfAH+aFP3t9T+4Zh8C7JItgfNCE - XVLMsKFG19UxRN3jWnfEU0DEw+TnrfK4ARiPoasHdYwo9RwdfpOPQKJeeXMB1l1jAYzH7R5+ERnc1TP0 - L6voXP1adScRd1GevLw8/SL4WcgXhkx8+MQXjsx9mV1/p77a5u/IzBdGHucr3blfvgolT/Bc7FCs5Pin - Qo4JzJUKhCPTBMsd7CCibvOX8hwRxbJRdTjNyigekpHuqiiaUqp4xnFW5vH08fZwyd1EFAsvuVFG8eCS - u6soGl5yo8zl2dFksOBuGoIEF9uoImhood1FBAsuslE10o6nfI83sq5qpJVJJs00JKQEF0zv83UEEUvc - 82QED0sk8mRT3k6ajklICS5ckju2JPMVKaG02qNLyyGPlUMuTAkNlRQVSwn1dQRRUqPyWI3KV6WEcnre - QVjKTErofTucEhoqKSpaO/JY7UBTQh0RwULbrJxrs3J5SigpJthwSmiojFGFB82mhN73kKSEkmKS/acQ - +ydDhFNCQyVFlTQITCuApIQ6IoIlTAnl9JQDlhLq60gimhJKSAmuKCWUVnv0NSmhLIDzgFJCCanLFed5 - kmKXvSLPk5F7fFmeJyF1uWie51RDk5BvL32dR5TleRJSnwvneXqygAcmlLkqjgZ9h01IPa4kQSUQRpjw - hecTVMLNyz/DpbQhGU1Q8XUBEfzQ3VVxNEGRkskh3ja4MKnkkNsm4PPviSTgCJqhMM/T/BvO83REPgvP - 8/R1AVFUCek8T38Ler/weZ7BVuyeYfM8h42CykLkeTr/xk+drSmSPE9f5xHFeZ602qVL8jx9HU/8KkV6 - PQ15nietdumyPM9QyVN/k0J/85honqcjcllYnueooChoBaLyPCf/x6oOked5+/cXlPOFYEhO7gt9bpPE - zN/qfSMhE4h5H7xAQ0LUZeWZzJ7FujOYPfq6zNeewRUx77PuTAYC4SLLWmXks3xRacWyVrmdBKUVyVod - 9xEdP3PEkmMMjgrOWnVVFA3NWg2VHhXueFG9LlmXi+tviTpbTE9L1rvm+tYrGsdYuyhuEiOtoeSFlnmb - 3UhHCjb8SMFmzUjBJj5SsFkxUrCJjhRshCMFG3akQJq1SmkjZLwQyKzV60ZB1mqoJKhwW7RhRkw24hGT - TWTEZCMdMdnwIyZ41qqrcmlI1upt/5CAZa26KoqGZq2GSoq6PBx1qiFIaNZqIKSYQNaqI6JYm99x1OZ3 - mgT3JJmsVWcTWMforFVnC1a/yKxVZ0O/VSKg1hFEOL01VMaoX+XYrwQXHQYi0lvv/8abaDK99b4BSG+d - amiS7N4O01udTZJ7O0hvdbYI7m0/vXWyAUpv9XUEERwoD9Nb7/8F0lunGoIkuQZ0+QvKnix3SXsStCVd - IW6gPCnNNXeNkHuV0lwh0+M15kcBvDPtyKY8JZ8Bp2Iz4JRwrpdi53qpNfOpVHw+VS+b+9Vzc79ehb8m - vLK/JrxKf0145X5NONnPIP6LZRU4ognr701X1ge9p+60f/3e9X++LW57KG2c/PvyhA5GPuH/0Ra12Vxk - qqm/9mbvf2R9ttiA0XMO37LqsvzLWkobJyNlQ8tH/jn/nG6rZndKc31G5jO3YvHHK5R2Sn65bs3UWUSn - 9aNDMyyHiLaUnmzktaedekrSsi+6rC+bWqXZble0fQZ8BhdjBE7mA4DD8ovpqgJauy3Sot51Hy0WUMnI - Xf4X+9Wg+fi1yO3FQOiB2Ge3WaeK9FhkwP0RKl3qz/aM8sKeEQJ1hBPmeds3p6I2ieJP+s4s68UfehJS - jruryqLu7TXGYysWoDhfXXzlazHurPTpF73MmGZxzvpWNnWlQKLteQLv0qdH+7G2+T5bN+BSKw/D+ZVK - XYruIdeRRHG+na4JMhuj5Kim6sqoRslRL/WKWnQV0+xEXj+TNMp9WP1MkPqZPLB+JlD9TFbXz2RB/Uwe - Uz+TpfUzeVz9TJD6mYjrZxKpn4m4fiaR+pmsqZ9JpH62qpc+P0cpx31M/eRRnO+D6meExTmvqp8BgXdZ - Wz9pDOf3mPrJozhfUf28KzmqqH7elRxVWj+n4gm7qT7SzXckEWEiGTkmQs5c4ZO2sNlH28t+X5h3Zv16 - YV6DFh/wPGniKlltqaNXW+ruCydd8wyBmkVpXbL+MzOf3rfDj+lpr09T6bM8IxYshPayoUVd9iaxuGk5 - 8o9CRv1RuMSyfs2qMgdbslDpUuFP8x2Rx1pzxWauVLBZlI01T3Jd7bWVGgVil70i4ouRk3x9Z6718BGO - z4/06VPyOT1k/bHoXmz+FmBBqCm6Sa+SkW9Kilrri590RS5EO3KKr7clZich35FTfLXL+l5e6I6c5H/v - pOircqSqpBT9GuLrCKLk1xBSPGEfs6dg6BYJfWEBCzyS1SbJnMvykBhOP+eABNHwhDkXKKImgnB8TNrU - ymvPIeZ9oFJjCPMu4NVhGfNO6BXiIY6XWSFg5TXiEPM+YOmxjInTSb96FYs7itfdHX1d6If0paoAxk3i - cpavqTLs7ajbpgXUem9fjZbDTUJy0uJdgNIql3ZRRwSjd3f0r+ZXRQBg958Q2neb6Z8uDjceFS7FrNtm - 3gDarLRZ4x0CDMQuW3eklX4vuA7IlAcE7WsJMjJA4Igo1gn5UdGTEbxe3zMmZg8m3oQuUzJe5et44m3E - bPkoA0/wXXp7Rvp1MwfqXaB0qccevvZXScAZ3mZA0iByWXY5ymNW1nAlcpUhdUimFEDvwpAprfC+NiRX - 2Uch447KkGrvBAn0LmSYx6I8HHsRdZAyXPh+V5H73W77aAuYpzUeCaw2YZ3p7V21RyBXCcU54pwjyTmr - gwClVRSt7QTnp0UMS3Rsg44i9iec1p9IUiUgVR6pSS9l3f/0GULdRB5L8NCkn5cD3fhURY39DsLIXT7+ - 2KCeGW9NL+4f+VqaDPZpJjKChzYed5HLej8r8Vn7WoKMHuVdNLJek1I0T9XX8cSvUuRXngm82BDSCfc5 - zUyXrlzcGxwVLqXqEULVO+rtrqkVoLf7O4Rd21QIwe7vErrK/FCSA8vuuqqABrxJj4qA0tmZqSBoEPms - HKO4Vzgvqj4z/wYgd41DKt51x/ICYAaBw9Dv6epYqB48oKnM4ZV5C2D03q663jeIXO/u6Y/l1iSE1x/Q - YUxkDs9U0IvKDsidfNc4pDo7m0XfatV3mVm8HAD6Uper0jJ7SatSIe3GROXRdkWHgYzAYTQ71Zq5yPoO - Qa7BVBby6sb+1o3yrjKHpxuscvchvBahmGKfs7Yt64MAfFM6VAVWCxXUCwU/m1TwbGp071ow5dHXkcRV - k6nmOKTjumlUsyDSUzIgxchJ/qqpTHMc0hGZxOTJSB7SD/VkJA+cuBQqfSo+pdDXkcQH3P9LZhJO9nzE - /b9oDuFkV/n9H5k9ONnhAff/knl8kz3x+5+YwTfZgN//xNw9b8OwhlzbNc3+vhgoPrsSgpLHIqqL9AzC - 1zYrVLrb7m7fES2G+sKA2XfPyf3rJPtjowLhBMF3Ab8VckQ+S1QCzNmb8c+rDVRHKTHFvpWKiD0Rj+x3 - 4YJm7+x6ZtcthwJZYM8RUSzTjthmBF38MoKgfNqn9skMwbUJbjBqo+TnFeRnkvxstu0y3VUXFPhUTdGH - 1smsQYWzR22cDC01zwIWeJjF21b7GMiMlzpnVYUuPT9PIl2XrzXsiChW30CP/EAYMOFJve/smobXLWoH - rgDt6wjibRXrXnB7eOoJ/eXTL9+e7fe0dh7F0FYq+036Yo8Iw3W6TmW3Pa986FzoA6u22fJ3/hmM55eX - BzN8ZfsyWXVoOr3vGbIiCbTLdfov8q00I/f4bWeWP7WTsc0YP5TXzgI8D/uhQW9/f9L7QHRXSnCNqWm9 - +3eYO0pdrhkVT8q0bJHHt6cLiMNzV9sdi3cQOpUGXPvYMsOyRa1KYOiekYf8pt4P44fnrNf7wga+PnDQ - ZwUv8U5IA27VNCeVVuWpSPNa2WMA8QThr3/5P8wvEPid1gQA + H4sICAAAAAAC/2JvcmluZ3NzbF9wcmVmaXhfc3ltYm9scy5oALS9XXPbuLagfX9+hWvmZqZq1zmx08l2 + v3eKrXR04tjektLTmRsWJUE2dyhSISh/9K8fgKREfKwFci34rdp1TsfS8ywKAPFFEPiv/zp7EIWo0lps + zlavp38kq7LKigcp82RfiW32kjyKdCOq/5SPZ2Vx9qn5dLG4OVuXu11W/39n4vy3zcXv24+X64vN+erd + hXj3cXv+bn35fvX76lKkv23efTzfXm62//Ef//VfZ1fl/rXKHh7rs/+1/t9nF+/OL/9x9kdZPuTibFas + /1N9RX/rXlS7TMpMxavLs4MU/1DR9q//ONuVm2yr/n9abP6rrM42mayrbHWoxVn9mMkzWW7r57QSZ1v1 + YVq8atf+UO1LKc6es1r9gKr5/+WhPtsKcaaQR1EJ/eurtFAJ8Y+zfVU+ZRuVJPVjWqv/I87SVfkktGl9 + uvairLO10FfRxt3313v8aL8XaXWWFWdpnmsyE/L465ZfpmeLu8/L/zOZT89mi7P7+d2fs+vp9dn/mCzU + v//H2eT2uvnS5Pvyy9387Hq2uLqZzL4tziY3N2eKmk9ul7PpQrv+z2z55Ww+/WMyV8idopSvd99e3Xy/ + nt3+0YCzb/c3MxWlF5zdfdaOb9P51Rf1l8mn2c1s+aMJ/3m2vJ0uFv+pHGe3d2fTP6e3y7PFF+0xruzT + 9OxmNvl0Mz37rP41uf2hdYv76dVscvMPdd3z6dXyH0px/C/1pau728X0X9+VTn3n7HrybfKHvpCGPv6z + +WFfJsvFnYo7Vz9v8f1mqX/G5/ndt7Obu4W+8rPvi6mKMVlONK3SUF3y4h+Km6oLnOvrnqj/XS1nd7fa + pwAVejmf6Ou4nf5xM/tjens11exdAyzv5uq73xcd84+zyXy20EHvvi81faedTRG+u72dNt9pU1+nh7qW + 5iqmc5UQ3yaN+LOdG//ZlP9Pd3PlVLdPMrm+Tu7n08+zv872qayFPKufyzNV9Io622aikqrwqMJfFkJl + Qq2LmCrUO6n/oEVZre9WXeLK7dkuXVflmXjZp0VTCNX/slqepdXDYad88mwlFCyaQOru/c//+J8bdWcX + Aryc/5X+42z1v8GPkpn66fP2C0GH+cWz9Ox//s+zRP+f1X/01Owu2SaqloGvof9j+4d/9MD/thxS1FRL + h/Se6+XNIlnnmUqqZCdU9bAZq/NJx8rQgR4pqidRcXQW6Vh1XZisDtutKm4cN8DbEZ7Okwt+yvo0YGdq + UR87pX3as8ekRDgdHlSZrrOd0C0bzWuQnvVRtXC5YIpt2HOzEgH59TF5Fs4xXVdkRVZnaX78Jcnm0NW8 + 1EC4qo87nc+TP6bL5Gb2aazfQHzPfDpZqJaKqGop25aX6SbRX9Z9LtVBpDhdtjff3U9v9Qc6ZSgVucv1 + xvvpt6QSXbyF6sTMxv9+iAXMq6yMsju8HeG5Um07V+/BkDvi8kFBH0P/8Wp2r/pTyUbIdZXtKTcKTIN2 + XWulB9X6FNmGoTdx1L/SfSieW6Ood53t1agj4sp7ARpjkz0IWUfE6AVoDF3By8f0p+i+zIzkatB47N8S + +A0/X5Ii3QmmuKODdvZVtzDq3qUviWq4JO/+cgx4lKyIjdIb0CgRWRBM/321jciAjg7Yy7pcl3kSEeFk + QKPEpX4o5TOZpKo1Ypg7ErOu8nL9s6uleHbTAEaRtao10mrDLToW70S4+3afpJtNsi53+0o00zrEruWA + Boi3rYQAvinJETEREFOVj3f09LNI2PomPwTxIBGzDStAtkF83GSBUmX5ly4H75L1Y6rqwrWoaC2lj4P+ + 8zj/+ZC/+cTKkTR/YAQCPUjEdsh7NWGFOcKwW7zUVRqXZJ4DjiTbn8kJ0KG+d/0oVP24r7InPWP/U7xS + 7Z4AiNH2MtVve6jKw54cwcYBfy7Sykg9SY7gCrAYbj4xI3kaLN6u3AheCE1i1rIZDTGvvYN9tyjSVS6S + ci33ulHc52p4Tg0BOdBIMnsoRFcL6GkQBez2khkSlqGx61zq/CsKQe60YRI/1jY/yMfjrUv+YTYN2FX7 + TnYqxjc1jbhOuWybrVUtQLW6PBZB3y88tyZDVt7N7PJIhH1apTuWuyExa1vjMmpsBwf97Y0ga/2sh643 + aMTeVOmSpW5RxHtsqpM8kzVLbxngKOpP6SFXg65UymdVZ6w4gTzJyFjJQYpqk9bpmwQ92eDo4iXhhupQ + 1FuIZ9Wkb8QLU37isQiRLTUogWNlxbZM1mmer9L1T04cSwDHUDdqXj5ERXEUcBw9ldPcvdwbyBLgMZoJ + C9aUBCZBYqmsi4/lSpBYjN7akYONxWGneiPrn4JXfg0c9jN7ggYKe38dMv1o/PFQb8pnVpLbBjhK8wQk + faTOPHk0bO96Tup+UUMcdt76Fjga8ckogCLeXKparCsFugpgZbZvgaOp2yPbvkbVUo4iGGcj9vVjRJCG + D0bgZruB+/7mGWb3jbxcp6x7EJT4sQqhRjX1bp/MF+TJD5OFzM904bPvqcSufBLcyQ2b9u36gyRdr1VO + U9UGGvQmD2W5iZA3fDhCJQrxUNYZY3CFaJB4bTW1PeQ5K06PY/5V8pjRGzOTxcylGkeveZncsWEzP5tN + wUCM2IwGPEjEZrDTZJfM/uYFsxWBOM0XV+wYLR7w67FAhL/FA/6ukokIcTIgUdg3ReCO0AuJBc/aoohX + 9SpXxMdxNop4ZXyJlGNKpIwrkXKoRMq4EimHSqSMLpFyRInsepW88nOEIXf9rlvomezLktHM2DwSgTVX + KANzhe1nx8khyVOfcMR/7Puy595gCxjtnJ1G54E0Up8dqidOrXNCg17WtITLIxHE+pE1QLJgxN08uUqy + DU9+okP2CHXYy09zg0cisObGexKxyuwhzR94CdKxYTM/SUwBEiPu2RKgQOK8RW1zPrK2SdRwvnxODsXP + onzWD+r33YwaJ5NwGRY7MtoYvxS57nhzWmTXAEdpVzuw9B0a8HLzfzDfm88jp4UwDxKxma5Piw1nNYMn + QGK0SxKYtYCJI/6o51hyxHMs4zsxBcsyIFHK3T7P0mItVIctz9a8PHElSKxDVekL0v1P7k+yFVgcVeR3 + XXnkRTEEcIzop4xy3FNG+aZPGSXxKaP5/e723qf1o4yJa3qQiKVsanRV3zaT87y0dSVwLJFW+WvzLLRb + 98Fp0gELEo33xFaGntjqD7dpLoVek1N1za/YJN0L0E3rxQk45ISv5KESqcIi0tI2wFGinunK4We6Mv6Z + rhzzTFfGPtOVw8905Vs805XjnukevyaFap+3VfqgX0vmxrIkSKzY58dy3PNjyXx+LNHnx80nMq54mfxw + hCStHmKjaAccqdBPINtUjOprQ56hiDJJN096gZoUm+iwjgyJzX/yL4ee/OsvNEssKyH3ZSFZhc4SIDF4 + qwtkaHWB/lBvknGohV6eIwrJDeFbkGj90mbOyxuoBYkmf5561RE3LqDB43UvLsfGczRIvG4TFU6MFoW9 + vw7ZOiJ7DBz1R6xokSNWtMioFS1yYEVL+/m6rDb9u2IRLRqiwuLWekRdFqoHKx/Tiw8fk3Jrjh0l7xKG + rNjVdOMD1WdX9ddhJ3jRXQsc7djE9Kubme0HKMJixq5ckiNXLpnfy/QLakWtqtOYaL0lHE1XOJtHwV03 + FVAhcaH3A9gdatyGR8+KB/2CU1mpEdKu2VFLckMDKiRuVe/1Tb7NcsGLZgqQGHWVraOn1HwLHK1bwqZf + Oo1oLnwLFo1dOoOl0Z7fjxkLwyY0qu7Etu28fj2R2+EHRWNjxnRTcFs4ep3WBxn7a0+SMbF4jYTrCEbq + V3PGRbM8IyPKN4kng9EOenJJ1T8RoY4KJI6qszePLH1DhqxxxdxW4HHEmn/9msXNlUy5YoUGvdFJYzqQ + SNWB1ww1IOzkPywIPSXoeqFv0DGATcGorPXXcnD99UFPLGyp3pYCbOoevm9H31/pDwRtesieTBa353Eh + GsVgHN2fioyjFXCc+WISl2CWYEQMdrL5ljHRuInnW+BoEa/COvign51yrmM4UvtYnJt2sGk46lvEwyPp + oV+7UWr9mjxm9CcJoMSONb36knyd/ljofRgoepNDjNRXuC0QcT6mMtkc9nmXVWWxzR6Iy5CGXEjkXVrJ + xzTXEzvVa/dtyYoLmpCoxNdYTA4x0psvB7W93dZ4id40+vR4tH8cTIkzoILjGk+e1+leDw85IX0LHI1a + pE0OM5a7ZPVa0yYwfBq2t3sAkDeoAvCAnze1higCcdgPhXBLINpeRKSZhgfcZhsgowJZpqGo7Vx0XLzW + EYj0NtORI5WB62jH4uyYLY76OatZADzoZ+1DgDnwSLQW1CZx607v915RFzrCBjxKzAOjkAeP2E3x5NlW + NOvwqF2zIVco8k7wI+1E2EycCwZw3B+ZOcE80R25yMrNUeBx+FVKT8P2TLaP6rh9GJOHIxA7kwYG+5oV + 9ryqo0OD3phehaNA48TU4XKoDpdvVDvJ0bVT//SHGydUQmVEDSSDNZCMq4HkUA0k1Vgi3yQr/eZl8ZAL + PTJmBQI8cMS65Pfqj2zYnGzLKiKzAQ0cjz5gtEnbSt/sANrjIGKf0eAeoxH7iwb3FtWbXKb7dqpBP9RX + BbamnC0QcviR9Lb17Zsvh9W/xbqWOrNVh5n2TCJs8qOydjEN7GCqP9JzY2/0UwIqJ26uv6Q35u9OcSBF + cuEBd5KXkQEaAxSlmRvoHmXojkFe0+P4DihS/boX7LQy4AE3M61cgx2lXT/0mJES5wS5Lr3aKm+W7zP3 + rEUUThy9fKzd8JTk7jHHF7PL7sAOu/SrBK4vZgfdgd1zeTvZYrvYsnewDexey9g6BtwxZn2o68eqPDw8 + tu+rCdrzHwC3/RtVbB/0KYvJuhLNA4c01/0j0vgAlTixyv44DZLe4Byj6qwwXmg0MNvXziif3htY1y/9 + Um49oqUEGXJBkZu57LbrRMsBAEf9+k0l3RMhV/2Yw4m0fuT9BINzjJG7QA/vAP1muz8Tdn6O3vV5xI7P + oqrUOIF52JEHO+6XfVk1S6Z0G71Tt3+lbntSANBgR6E+u/Gf2ZyOjtWLyZqjOyg+n3bt9TvzVXtamfdp + wG4+dtbdIkmO4BmgKNSdW7BdsGN2wA7vft18qquJZpVlqXq4VUbrAcAGJAr7mTFsAKIYr42dtlajlx/Q + AkRjP4kbegLH25Ec2428f2IVO/YOm7Co3Cd8Y57s9d/pukzdCSPt6jhmOFCFxXVX5DFjehogXvfuViV+ + HVQDqJpD4h5XqASMFfPCCKKA4rzJM1LSs9GHZosf+k6mJucZk26xEVF4xHyf6uaeTv5TdSs1oz0eiaA3 + 3IoI0OOwv90Ui+03cNiv8zytD5UwlsSyo6EyJPbxULHYbAJFcMzusQc/liXwYzBXRToo4G1/2eo1eUrz + A91t46ifUW/gbyMxz8BAz7+IO/ti6NwL4/NKFadyx5S3MODuttyhL6Py6YC9PyiMHaJX4HHUSCktYqKc + BGAMVSlmG4a64TAj9ZA6m/Stx514GE8cAdz3e7Mb1AieAIihh9Rkr4YAF/0ZOLp+yfgg+evDu9+TxfJu + Pm1WI2ebF2YIwARGZa2WCq+S6g5a2clEHvZ6koGuNmDfvSXfLVvgPlH/yOSjoLs6zjceN/WkGo8cZuTc + yz3pW9k7IQ2cbNN8/ERu/xTie04TPkkuyHWBBftu9u5JA6fhRJ+EM+IUnOgTcEacfsM5+QY+9abdi/04 + K0I/LBLi/QiMZ0foeTfNqsbjNAJrWs7FA35m59nlkQjcCs6CMfdBD+jikshxIJGafVxq1dGUzXR1M2Ul + WfFAExIVGN2xYgIeKGKx0XPwvN6yTQN21rGCNglYjVekyF6DDZvJy4RBgR+Dv/fP0ElWzdEQq6ykOjUD + mFi7B4XOwjp9JvWcXrEWLPERBtz0zlkF9c6kWOu7pj/1pJk85nUnQy4ocvssyNrphB4SkECx2vlV1hjc + glG3fj2ece/bNGbn9Ex7MmRtnpTx1Q0O+VmzBeg8rnxMK7HhTvzYNGpn7H3v05CdV/vh9R40JbrJHgS9 + k42bxkXVAwBWAQq4xkVm3RGIB4jI3b3pIbxzk/FWTfogEvmT9tYDgAN+9lILn4bthyL7RZ8u7knQauy+ + c3oIywgBaYbicUqwb/CjRGzeP3ieY8xZjuFzHCPOcAye32h8SF/y68Ggm9PmoCPzZ0bv8hnsXT7T+2rP + UF/tWVVZgt2htGnbrt//il2HgDn8SN1IiirvMNuXFcw3+i3QcxobrBOlBulZ1VifqtOI45HJRtU+JE+L + eB4tZ01fuKxnbnuIRGUL+S6g2dYbUe0lNRECJjuq7osc9hvinFFP2bY8W1Vp9UrOfpNzjPoI2/7BI3Xk + BOCAv10Z2S5+lWS9Rdv2XfqQrU/zKafNRGtSeUElbqx2QxO9UK1dokYL4tKuXW+Fr76gF9lRpw882HZz + zx/Gzx4mvmPrvVurt0a3BvekUuHTtn0vBKmLpL/vGsjtCtimqL77Wp/F2Exk7ktZ8xb0BzRwPFVFn79v + HvYdizP9Fcohlxf5KduI9hKpLagH2+52Y3BVxk+/Otnm2cNjTX3SFBQBMZuZs1w8iZwcpUcBb9uB4okN + 1jZXxEqj8uoJ5sHH6DnHxgecOwrAXX+zyNHITT13LGkxQIUbR7rLFf5NfFcJUdhxuu3F+/XJlAge7Lr1 + MSsqct6+MEhT26xr1m8hZH+LdlOpLM/qjDbVARuwKBG5jUrcWG09Vwnqi1026Vo5bw1g5+FGnIUbPAe3 + +ZD6OOQEAa6oEy7HnKXbfOeZc8XP0BWfs/LoHMkjzlm86Dm8MWfwhs/fbT6F3kokh4AkQKy+G8z7JQ4P + RGCd9hs66Zd5yi96wm/M6b7hk32bTx9LhlJDgIv8rgp2OjD3ZGD8VOCoE4EHTgOOPAl48BTg+BOAx5z+ + K3lvL0js7YXmrNzmvdNmzpp6vRYLmHnnBAfPCO4+lM1OsXogsy43Yl8SFyrgFj8avTVKoLaIcywsetZw + 1Lm8A2fyRpzHGzyLN+4c3qEzeKNPxh1xKm77lWajAt7tYsGAm3sK7sAJuPGnpo45MbX5Tvtatm7R20NB + yUFcARRjW1Yqh/QUbTO3KtMHRhxAAsSirzNH91iT5LXTElg7rf8WNWqqh8ZLddNz2ObpA918BH0ne9Xz + wNmv+uN/b36enyfPZfUzVd2ogpzGLu9HYK9ZHjjtNfqk1xGnvEaf8DridNfok11HnOrKOdEVPs015iTX + 8CmusSe4Dp/e2nyjPpCl9cH3sF+KHzivlHlWKXpOafwZpWPOJ40/m3TMuaRvcCbpqPNI3+As0lHnkDLP + IEXPHz0dHmpucE9/qz2gQeLxshs95/T0YczieVSCxNKjGT1ls37lD4tQERiTuZJx6PxW/tmtoXNb28/6 + BxGc1sTloQhveTor52RWSV8JLqGV4JK3Zldia3bjTzcdc7Jp851HsTH6ufRH/KgEisUr/3jJf5uNNijn + or7Rmaijz0ONOgt14BzU9vRSxugcGZXHnac65izVtzmBdOzpo8ZxjHq8Rl4zDfFohJi1u3Ls2l0ZvXZX + jli7G3kS5uApmLwTMLHTLyNPvhw89ZJ74iV+2iXzpEv0lMvYEy6HT7dknWyJnGrJO9ESO83ybU6yHHuK + ZcwJluHTKyV9nbSE1kmz2mi4fSa3LECrov/E2IPU5HAjedNpD7bddVk3R79xV/hBvB2Bf6Jo6DTRyJNE + B08RjTxBdPD00KiTQwdODY0/MXTMaaHxJ4WOOSU04oTQ4OmgsSeDDp8KGns25/C5nNFnco44j1Ovjkoe + RZ6X3Z6f3To8YhjQYUdizCuDM8nPKS0R9Pddg+wfGyVZ8ZTmtPUEoMCJoReHkpwasBxPF++P0wTk6S2P + 9cwsJeLq5hhZSovtzcubBe/He6DtpMsgC+sHe6Dt1CeQJqvDdqsKPcMM4Jb/6Tw5Z6eoD/tunhSzcVPY + h133RUwqXIRT4YIpxWwRqXARToWINAimAEcImyJ+O/LLNxdZYpwXNdbpYKiPstYIQHtvdrHhXKeDoT7K + dQJo71U9i6v5j/vlXfLp++fP03kz0G6PU94eivXYGAOaoXh63/w3iHfSBOJthNg3F8YOdTIEougVbcUh + z9lBjoJQjMOOrz/sAub9QT6y1RoOuOX496YgNmAmbZYL05Z9MV/eq+/fLadXS33fqP/8PLuZcvJ2SDUu + Lim/A5ZR0YhlIKSx4+lVsLP7L6c6Yren3vmYAoujV9HXghegZVHz+O38PBBzqj9teFJNYlZOofVp1E4r + mhaIOakF0CYxK7WScFHL22wxezv5NmUXZcQQjMJomzFFKA6nTcYUSBxOWwzQiJ14I9kg4iS8qu1yuJF6 + Y/ow5ibdlhaHGPflnnQoEggjblrPwOJwY9xNaQqwGIQN+TwQcVIrKYf0rXE39NC9zC3CeOllFFywzHKL + K15S5WO2Jed3A/kuVjY7OTy5ulLDuuR6uriaz+6brhflByN40D9+sxQQDroJ9StMG/bpIrn6Nrka7eu+ + bxvWq3UiinX1Ov4AagdzfNvV+cUlS2mRjrWuuFaLtK0bQdZ1iO0R6xXn0gzM8TFckKdk50UZyAvZHPfQ + fEB5LwxAfW8XkOM1UNt7KJ6rdE9V9hRmS/bpZjN+ARUI227OdcJXGXGN+BUubs+Tye0PSv3YI47n02yZ + LJb6++1hySSjC+NuUlMBsLj5oXkJs+bKOxz389UhK6X58dGA97BLVq+EI/1QAR6D0H0G0KA3JiclnJPf + 7tlF0EJRL/WKDRB1kouHSbrWu7ub6eSWfJ0nzPFNb79/m84ny+k1PUkdFjc/EMuYjQa9SVbUH3+LsLeC + cIxDdJDDQJSMnUChHKUWPBvFvZKfnzKUnzI2P+Vwfsro/JQj8rMuk0+33AAN7Lg/M2/8z+id/8f0VsW7 + mf3f6fVy9m2apJt/k8wAPxCB3iUBDQNRyNUYJBiIQcwEHx/wU29cgB+IsK8IC8pww0AUakUB8MMRiAty + BzRwPG6vw8eDfl65wnog9sfMMoX2RGaTD9xUsVHUS0wNE0Sd1FSwSNd6u5z+oZ8m7vY0Z88hRsIDQpdD + jPQ8MkDESe3WGRxuZHQAPDpgP8TpDyF/xkuODEsNclntOcQomTkm0RyTUTkmB3JMxuWYHMoxejfNIh3r + 7febG/qNdqIgG7FIdQxkohamI+S47j799/RqqfcVJCzZ90nYSk47g4ONxPQ7UbCNmoY95vqultN+so3Y + fLhwyE1tSFw45KbnlkuH7NScs9mQmZyLDhxyUytYF3bc9+rvy8mnmyk3ySHBQAxiwvv4gJ+a/ACPRYhI + n2DKsNMkkBr8dABSYDH91/fp7dWU8yDBYTEz1woYl7zLXCJX2BaLNmnSzYZmdeCQe52LtCDWp5AAjkFt + BdD6//gBYX2Uy8FGyoZ6LocYeam5wdKQfPvjtWL/QOkd+4efYNSdqD+nh1xv0yZ/MkNYDjhSLoqH8W93 + +yRspVZgaP3dfUCfkjLBgDMRL2ytYsPmZLuPkSsc9lN7Emgfov/gHVP4DjUmq9fkdnbN9HY0bo+9O+So + u8P9VpLK9VtE0x44oho8fl9+vuQE6VDES9g9xeVwI/dGP7KOefnxnFtd2yjqJfYsTBB1UtPAIl0r81nO + En2Ww3qAgzy1YT6qQZ/PNB9ssu2WrtMUZKMXHOS5DudhDvwEh/XYBnlWw3xAgz6VYT2KQZ6/nJ6W7EuZ + vbCMLYp5GQ9zwk9wnE+b5bAx+kYAxVBV84MoRNUcbrPRu7bRw/gOJBIz+Y8kYtUBk5qlbVHX++N+Sh7Z + HCHIRb/zjxRkoz7AOEKQi3zvdxDkkpzrkvB16dMpWLJzx/b9dvbndL7gPwuFBAMxiFWzjw/4qZkG8G6E + 5RWrMTY4xEhvki0Ss+72nLvexxE/vZQYIOLMeNeaYddILgU9hxjpjbdFIlZqtWBwuJHT4Pq45/98ya4m + bBY3k4uBQeJWemEwUcf752wxi5i99/Ggn5ggLhx0U5PFox37JnsgbDVlII6n7S3VInl6T5IZnGesk3JF + OVvSwRxfVotdsrnISLYjhLgo+3h4IOYkTmQZHGikZ7DBgcYD5wIP4NXpg144WdJyiJF8f5sg4swuNiyl + 4hAj9U42OMjI+9HYL2b9XOS36g1sWPdJB2JOzn3ScpCRlR1IXuxTYg/xREE2vSE43aYpzJas6xeeUZOQ + 9VDwfnPLQUbaXr4u5xh3q27OgPw0ziIxa8HXFoC3bb5Uev9Nu6MNzjGq3uwuq7MnQa8mbNT1HupElLRZ + +o4BTIzWvsccX50+XFBfe+oYwKQyi2xSjGsSu33e7DNKzQSLNKzfl18UsPyRzG4/3yXdK9UkO2oYikJI + W4QfikCpkTEBFOPr9MfsmplKPYubOSlzJHErKzVOaO/9NFnMrpKru1s1JJjMbpe08gLTIfv41IDYkJmQ + IiBsuGd3SbrfN8ezZbmgHOgAoLb3dBLZuq5yitUCHWcu0iohnTDoYJCv3TiYaTVgx603Kyr0qQ3NV0hm + G3W81OT0U1H9pRkuNscdETddRgVIjGZv4eThkFZpUQvBCuM4gEi6HBImkVzONm7K43mrFF9P2TZRbika + 9XWb17s6kR6sW5Djygmbk50Ax1HRctGpJ7u/JGmeUy2asU3N6iPC4iiT8U3EM1sdDPTprYJUVoxf/wOx + vnn8wRY9AVj2ZMvet2RFVlM9mvFNOz1dwsiAIwcb9+O7sA7m+9jZGchLZuvjoJhXH4U8fuN7iPXN1DNR + XM4zUn+482sfxcvmsCMV5g6xPTqDClJZbgnXUpPb6CNjm3QxbA6qK2gpZHKusX4kV+AnCHBRuqIGA5ia + LetIL/UAKOYlZocFIs6N6vJU5StL27GImXpDWCDi3B+YTg0izopwwKYHIk7S0RU+6VtLet/JwGwfsbB7 + 5Vw3AqusTPZpVhFFJ843MrqqBub7aH2LlgAshBNpTAYw7cmevW/RdeLqsKWqOsz3yXL9U5ATvaVc2wvR + 8+IaDruVqMj3o4GBPn1HqTaEoexI28oYooGjs31JKhDq6w6vFziQCkJLOJa6IjcrR8YxEYdke29ERq3c + /TqdWnT8MtOenCyLc6qmgQAXZz7KAl2npN2uDeA4nnlX9Yxck+TU3RKuuSWx3pZerS3JdbYEamx9/s+O + JlGA66DXrhKsW6UQP0kW9X3XoHqBOeGMegsCXCrzmtNvqaXIgxG3HkrsCXs7gzDiZnthJ3WsL8GZG8mb + uZHYzI0kz69IYH6l+Rt1TH+CANeeLNr7FupcjQTnamQ3RULsTxkY7BPlVs88HKqCo+1p314QlmGYjG86 + zYyQS0hPBqzEuRoZnKvpP5V7sc7SnKfuYMxNHrI5qO/lzC9JdH7pNDjsTqgjLS9ABU6Mx/KQbxI1RuOk + tAuDbnKR6zHER3woZXKgkV4QDM41tjmpPqMJT5jjK+i9/iNjm2pBe26hv+8aJKNp6CnbdtDH2pN+V0vY + lifqnOCTPx/4xEnkJziVnxmDxWdwtEgulEBpbG9+4gOrEwS5OMMImzSsN5Ov04tPFx8+jradCMiSfM4K + QgXmcKBxRul22Bjo+77fUOaJXdBw3iafbma31+2+E8WTIPRvfRT2km4th4ON3aG/lCQAadTOTIYskAqU + uVMbs3xXy78SMf54pJ7wLMRsOSKeh/AKX094FlrydIRnkXVaUa+mYSzTH9Pbq0/NKhyCqocAFzGtewhw + 6QeJafVA1nUcYKSl/YkBTJJUFk6MZfp2d7tsMoaytNblYCMxGywONtKSzsRQn65MZU15eRkV4DG2ZZXs + ys0hP0huFEMBx6EVBhNDfUmu57g2TG1HW/Z0JZNMJs9lRbEalG3bkCwbjyZfSIfYHrm+WBUUSwNYjlVW + 0BwtYDvUXzKSowEAB/G4F5cDjPuUbtunnmm9WrGuredc40asaSoFuI5HwvqcI+A6csH6YSfM93FS/Ui5 + tt0+o4kUYDmatasERfN930A5YMVkABOxceoh20VYBnRr7/HQ/ptaAx0R20Nrur0We10eCl1dPyd/i6rU + CSZJOo+27OqOodVtLWA7sieKIHtyaWo6HxHbc6DktvUmpvq3KB7TYi02yS7Lc/0gPG2qzCrbqfFR/dpM + uRD0Y3R2/F+HNGd1dxzStr5Q0kR926KJd6F3/22rcqe6RUX9UO5E9UpSWaRlfVhTior6tk0f37TWeSES + UuPgsY65Tqrt+v2Hi4/dF84/vP9I0kOCgRgX7367jIqhBQMx3r/750VUDC0YiPHbu9/j0koLBmJ8PP/t + t6gYWjAQ4/L897i00gIvxuEj9cIPH/0rJdayR8TyqN4Rrb1oActBevB46z5zvNWjDdWOEcdUPeS6CvGQ + 6lc7abIj5dpK0rCnBTxHQbwYBbiOffl8QZNowrPQa0mDgm3bVLVU+gkGT2vgrp9YwKFRq/qb7ijRLJqw + LLmg3STN9x0DedR5RGwP6aznEwA4zsmSc8uySyv5qHoqpHVhNub45E9qb/jE2KZyQ5yt6AjIkvw6ZOP3 + AHA5z0jrwXUEZLlo+lN0V8tBRqYw7GN1gWEBHoNYT3isZ24edkjqJXcUZktWuX6lZMOzHmnUXm645hIo + +eR6pocQ1zlLdo7ZWPelxSLmCDHi3R1yok4RkIU3+PJhz03sXBwRzyN/VUSNIiBLTdf45U4eVlTNYQVZ + WEXixHlGRnXl11L7jNabaAHbQSuXbplURYr6SzrE8tAeM7lPl4pCJQ+F19/3DdQ7oIdslz4Rm9aFOSKg + h5rAFucbKYd9m4xlog1m3JHMPtUtju78JYdC771Eag8B2rZz5/cCM3mk3TaP3/cNlEW+PWJ7pDhsyqRK + SWskDAqz6f/zIHjOlrXMxAv0rox1SYFraf9MG55anG2k9owqv1dUkXtEFdAbkmJ9qASxAu0hx1UTn/d0 + hGdhTL+YmOejzZVJYK5M0ufKJDRXRuvduD0bYq/G69HQejNuT0b3Rqhp0CGWpy4T50BxgtGHQXd3CiZD + 3JGuldVttjjLeKBNLhzcmYUD7UHmwX2SeaAVhYNbFp7S/CCI7fiJsUzEqTVnXu30le2hWNdZWSSPhBoI + pCH7T7Fepz/p3pbDjXqlTFmtuOIOD/hJ8+oQHHDLXwchCK9KIDwUQYp8S+t/+ajh/f45+Tb91m1HNlpp + Ub6N9CjUYHzTQ1U+U02agU3tKX4cX0v6VkrvoEd8j35ltnoiJ1qH2b6d2FGe7p8I2yLrimhpCc+Sr9Oa + qNEI4CGsDOkRz1PQf1YB/a4iFwXVk5tv9l99+tRMZVOm+E0GNiWrssw5ugZEnKRjvH0yZE2es/pRb37K + 158USJxyXZPPSkAFWIxs067DqAl7UuAGJMqBnxGHUE4c3iArDkN5QZogsSDflavRDP2uaSnfJvfpWlBl + DeS7DucfqSaFgJ7uBM9kX6mPXsZP5QQUYJxcMMw59NsvyGVTIaAn+rf7CiDO+wuy9/0F6GGkoYYAF/3+ + PkD3tfoj45o0BLguyaJLyBKdqZcj8nQtL5IV/Ze3GOCrt+9Zwo4DjZcMG5CiesRHrlEbyHYRT8c2ENtD + 2Uji+H3HkBFfhrYg1yXXabVJ1o9ZvqH5DNB2qv/Ixu851BOQhXJghk05NsrOtCcAcLTtuJ6cG7/vLgjb + 7maBnSq/CaHD7HK2kTJ0P37fNyTkOqinbBvxh3m/hzj6MxDbQ5kwOn7fNCy6gYCo9PzcRlTjZR4KebO6 + O8HiMZWU+XDcAETR/Wh9piWpH+6ztlnvCZpmhezeC3ilVFAQ7dr3r9TusUnZNlotvPBq4UX7wmfxShyZ + 2hxuTEQudoTdYjEejqBLYGwU1wFE4qQMnCr0MbsDIk7u7x/83Um22+fZOqMPqXEHFok23HVJxHrgaw+I + l3zzniDflaeyJnW5LQzy0cbKJuXbyr1+GkBcmQrCA27WTeEbhqLwJoeGTENReUUQcviRSDMQJwT08Ads + qAKMkwuGOReA64KcqM4MxOmP0b89PAPRfYkyA3FCQA8jDd0ZiAX19RkDAT36/Ue99IfhO6Kgl/Fb3ZmN + 7s/kahaqYWNmNjADEIU6s2FhgK+os1wNZypJ7iQYKOAlz5jYHGi8ZNicnKKNGhfeqHGhX145Low79TLE + A22YhDm8SM1WQ86whxgIUoTi8H6OLwjFUEMsvl/Btps08l64I+9Fu/ulfiWYYjlBtqtdPtm+9ppnf6v8 + pbyYgRugKId6zbQfSccqxM82iUmPfxzQdsqf2Z6i0t93DPX4p//H77sGylPsnjAs0/ly9nl2NVlO7+9u + ZlezKe30O4wPRyDUVCAdthNWLSC44f82uSJvumRBgIuUwCYEuCg/1mAcE2lnv55wLJTd/E6A45hTtmPv + CcdC2wfQQAzP3e3n5M/JzfcpKY0tyrE1u0IJSct/F0ScedntcM8Sn2jH3laqeUboQ9mY4ZvfJNezxTK5 + vyOfsQmxuJlQCD0St1IKgY+a3h/3y7vk0/fPn6dz9Y27G2JSgHjQT7p0iMbsaZ6PP+oYQDEvaY7XIzEr + P5lDKdw8NVFNK898pDE7pQfogpiTXRwCJaHZ+E4v72GnhGkYjCLrtM7WTW7r8Ua6FZFBfSF2DbR9lSHW + M3/7vpz+RX5MDbCImTQ0dEHEqbcMJG09DtMhO+1JOYwj/kMRd/0GH47A/w2mwIuhOqs/VC+D+sAeglE3 + o9SYKOo9NB2tZKV/nmQGsBxepOWX+XRyPbtO1oeqojwkgnHc3xxj0h1KzQ1iOsKRisNOVNk6JlCnCMfZ + l3qio4qJ0ym8OOvV+vziUk9+Vq97ar7YMOYWRYS7g333dqU/PufaHRzzX8b5B68/yo66H1P1v+TiHVV7 + 5Hxj25rpPiL1AB/c4Eepq4g0seABt/4n4UkIrvDibLO9TM4vPyYXyb6idkps2HeX1U91s9ViXev/Xotk + l26ekudsL8qi+VDvdKxfuKFM3TLc/pXRO/JgD745OpxXwEzU8z6sdzrrUnLnogcxJ6/mtOEBN6u0Qgos + Du+Os+EBd8xvCN9x3ZdYHS+LxczNiPCneOW5jzRmV43z+A1aARTzUubVXdB36uPcXtv+b3t8M7eXFTAF + o3bnML9FWFcVjNteaHxQywNG5FV7D9DZePZnpwPteeoTDvqbpqHbejUrC0YIxwBGaVKPcgoPxKJmvb4z + IotdBRinfmxOPFXfJUzrw7jvf0z1Om366LAHPade75rKHVHYUb6t7VqSe6QnzjM21ap8lZTdSQDU9zaH + tm6zjRpmZmmerA6UxfwBhxcpz1ZVWr1y8s1EPe+OMwe8g2d/2z9zLtEgfavYEfZMsCDPpWsnXs1pkL71 + sEs4syEnzjOWMeO9MjzeK4s1tWLUiOfZl/nr+ft3H3h9KYfG7YzSZLG4+UB7yAjSvr0SiVRVxap8YV26 + g3v+asOow1oIcemd2epsn4tLyrmvAYUfR3AqmY4CbNv2IAQ1WEl08GYDYdLLJUMiPGZWrLlRFOp5uw2Z + +BWnLxgRI2uX70SH6jxYxIPkxtAkYK3b16Qj+tigA4z0NuMXSRi/yLcbv0jK+EW+0fhFjh6/SPb4RQbG + L82R1puYqzdo0B7Z+5djev8yrvcvh3r/vE4w1v/t/t7M9kkhmNoTjvqzbZI+pVmernLBjGEqvDh1Ls/f + J48/N1u9ObT+uvqeoCY+YgGjMeZ7j5jhW86T6/mnP2inPtkUYCPNz5oQ4Dqes0L2HUHASWonTQhwURZT + GAxg0u+8Eu4AGzN8j+mVHsO285eqzL6Mnwf1UdRblI/PTK9GUa+UUrxnihs2bE5+e4mRK7z3X08Xxwnv + 0VdsMrZJrFfvqQM2l8ONhA1MAdTzMi8UvU7+ZeJXuREX+rEu61Id1jO/jzC/H2+mJoePO/6CXlqPjG0q + mL+/QH97wf/dReg36x4N4XGKgYAe4qX1FGw7FOtHQTm6FYR9d6kGKfu0ymryD+9Jw/qFtDN593WLb66U + IGi+7xuS/WFFyk6Hs43lbn9QQyqir6cwm56ZfiTkKQSjbtrpoyBsuSm9te7rFn86CY+WjCYG+1QpTHei + FpWk3HSYwIlRv0seSE4N+A7qb24R37OnWvaA4xf5FykE8FTZE+eHHTnASL5pTcz3/aKafrkOfdDeP38/ + /510ZiKAWt7j8VR9uSOYfdhyE8YZ7bdtmni2hIFYnvbFDtbvc1HLK+n3koTuJUm/DyR0HzRTLc0byzRT + B9mu7G9K/aq/bvG0BecnwHQ0qS4pp+KajGGazadXy7v5j8VSA7SmA2Bx8/gBuk/iVspN5KOmd3F/M/mx + nP61JKaBzcFGym83KdhG+s0WZvm6l5mS28m3KfU3eyxuJv12h8SttDRwUdDLTAL017N+OPKbeT8X+6XN + vPyeshwGhA33YpIsZsTaw2B8k27jqSbN+KauFabKOsz3UbKiR3xP03pSTQ3kuyQjtaSXWqTuRPd929AO + zPRmEWl9qEi/zkFt76aMUfu0Z9efEJUa8TxPosq2r0RTCzku1eRffyGJGsK2UO9H/15kDQUdDjHyBoOo + wY1CGg6eCMBC/uVeL/b41z3Zs4csv+i/y+4Nn/5KHRa6IOQkDgwdDjD+Irt+eRbqw2UHA33kZbEQa5sj + hpsgjdhV7jFuaQBH/IdVnq3Z+hNt24ntrtfmsge6AAuaeanqwaCblaIua5slo26TYN0mGbWSBGslybtT + JXanUpt1v00nDfW779sG4mD/RNgWescC6FUwJg1MqHdNr3hz7S6HG5tX2bjaBrbcjPGJTcG2kniKKsRC + Zsrox6YwW1LxfEmFGiXTCP5i4ijNA2HnC2W3DQ+EnIRWyIIgF2kE6GCQT7JKjURKTV1yy/aRdK3EcZYF + AS5alehgro9+YdBV6b+1BwoVeoF8s4Q4F+lPs33nvGPLs/tX97egRvzbK2mcZPfTPPnj8745UDNRParH + 8Wd2+6RnLTJZ7y8ufuOZHRqxf/gYYz/RoP3vKPvfmH1+9/0+Ibw2YzKAidCJMBnARGuUDQhwtYP4dn6g + rMhWG8f8ZUU4aQJAYW+7KeU2Tx846p5G7Otym66ZaXKCMfehehK6BPLkRzpop8xWIzji34gHTgnsUcTL + LiZoKWlva8JhNz4JWPVcxOo1Jpk9AxKFX04sGrA3KUaawAZQwCuj7ks5cF/qz/mVlUUj9mbXHv0yqWqB + pT4UWXUPdqxIoMmK+nX6o5tnp43dHBBxkkaZNucZVYZnqii128SJdTV+e1JU4McgtY8d4VmIbeMR8Tyc + aXwADXo52e7xQATdJFclOTl7EHYy5usQHPGT5+xgGrI39yH1XvZY0CyKdVNdSYb5xMJm2sSeT2JW8kQ8 + gnv+TCblPv11oN6CJ84zqvy8ILxSa1Oe7Thlzmq6YQEag3+7BJ8bdN8hTascCcjC7smAPBiBPDSzQc9Z + rusLeqp2FGjTKc3QaczztQ8R2Enq4oif/lgGwTE/u/QGns8cv6E+Y9zURwz2qfzg+BTm+bh9WI8FzdyW + SAZbIhnREslgSyTZLZEMtERNX5zRSTlxoJFfah0atnM7KDY84E7Srf5Q5bUaaGVFSppRHufzroD2yM2C + LNe36fLL3XW7yVQm8k1Sv+4pFSDIWxHaJXXphtKcnBjA1Ly/Sx01uCjkJc0bnhjIRDh1w4IA12aVk1WK + gUwH+u9zx2v0VaQWBLiaeb2Y2yekGR2POGEzpALiZnpSoSbHaDHIJ5NU766iNxKq6aXNxmF/WbSdGo78 + yALm3YFeohUDmGg9amC98OmvTddQz/6QfScSsDZ/J3abHBK1rlcrplWRqJXWJXNIwCrf5u6WY+9u+XZ3 + t6Tc3W1Pb7evhJRi8yaxcR0Svy751YHDWxG6gU22uSgIJ+p4IOiUtfpsw3C2oOVsTu89ZHmddXUPpZz5 + sO3W/ddEPzOlOE8Q6PrwkeH68BFyvb9kXJeCINeHi3O6S0GWq9kzUxWoNruap8Evu00iH1P9n1I+Hwgx + hmWh2OpnHr+u/zMuNiAzYl9ffPhw/rvuwe/TbPzDDhtDfcep+PFvUaMCPwZpbYjB+Cbi2gmLMm2z+8l8 + +YP84pYHIs7xby45GOKj9EUczjDe/jG7Jf7eHvE8ulJrF6cQ5/NgHPTPY+xz3N2c7XaskUXxoD6SxAiQ + wotDybcT4Vkq8aCaJFE1RzfoljsXNTULQYcXScblqRzKUxmTpxLL0/k8WUz+nCaL5WRJLN8+anv1xoai + qsqKNt/lkSHrlq/d2t52BqL5mOI0MMgnX1XB2XG1Jm3b259BO+bY5XBjUnCdSWFbm3Mt2o8kxWlyjvFQ + rNk/34Ntd/NMjppVJwhxJbn+E0fYkCEr+cYCcN9fiJf+W81W3dQQvsGOov7IzkKXdcy6Zfk0u+OUOZcF + zPo/uGaDBczzye01W23CgLvZyKpk223c9jcHWpNvmZ7CbOSbxkGDXvJtA/FAhDyVNTMxejTo5SWLww9H + 4CUQJHFilXs9ZNul1U+SvcccX6WXhTUhScXa5HBjsl5xpQoNeLd7tne7d7wHTok7gGWtEqksC3bFDOCu + f1c+ieZoVEET9xxo7DYY5opN3PXLuqxYl2yAtlOmnDToKcd2atCpt6xN+lbqTXpkDNOf98lkOrluzohP + CUejeiDiJJ5wC7GImTQOckHEqTtGhJUxPop4KbsPe2DA2b7ss8kqsaacjTTkQSJSRvsOhxjLveBdtAYD + zuQhrR8Ja+sRHokgBeE9RBcMOBO5TuuaedmmAIlRpw+k1x0BFjFTTtLwQMCpl3HQ9mIDUMCr39tUzUn1 + yKnpTBhxc1PYYAFz+zIfMz1M2HZ/0q9gLsuvhOU9FmXbrmb3X6bzJlObI5ppLxNiAjTGOtsTb3APxt30 + NsuncTtlfYuP4t66yrlehaLebpNlSk8TE6AxaKv4ABY3E3sJDop6m+Ur+z2tS4cr0DjUnoOD4t4nRoUC + 8WgEXh0OCtAYu3LDzV2Nol5iT8cmcWu24VqzDWrVh0Fwi0jDomYZX8blmDKuvxRTA5z4YITo8mhLgrH0 + ltv8CtMwgFGi2teBtpWbD3j6x9Q04VomKkcHcpJZs6C1Cu/e9+97ercH6us0f/ucFbRxjIGhPsJOfT4J + WWfUBvBEYTbWJXYg5PxOOhPS5WzjtVirEvQpleLjbxSjyYFGfdczhBqDfOSyY2CQj5rLPQXZ6DlicpBx + c0OuZyzQc+oeMScRTxxuJJZvBwW9jOw5YqiPd5ngfdh9xsr2HnSc2YOQtB/dEJCFntE9hvr+uvvMVCoS + tVJzxSIhK7nonCjMxrpEuNw0Hy0oq/csCrMx8/uEYl5eWh5JzMq4bRwWMnOtuPFP2tpIh8ONzNwyYNzN + y7Gexc3c9DVp2z69vbq7nrJmTRwU9RLH1TbpWAtWv8bAIB+5LBgY5KPmf09BNnqemxxkZPRrLNBzsvo1 + JocbifW+g4JeRvbA/RrjA95lgu1T9xkr27F+zZf7r9P2yQD1ca9NYtaM6cwgI+eptAUiTsYMv8siZvGy + L6uaJW5RxEutkS0Qcf7cbFlKxWFGseMZxQ4xcp/YgQIkBrFVMjnESH2ubYGIk/rU2QJRZ33YJ+mhfkwq + sc72mShqZgxfNBxTimJDm83CLWOjtUsd9Hs8rH1WGe7glb1Fso9L8ejEHpHO/z8lMSN1qSsSLBBwfr3+ + 3J7SvqNXQwaLmDOeFGwzv06/Nbub5IwqyGARM+dKGwzxmTsTc6/YcWCR+h1C2IEsBRjnB7tvYbCYmbhy + wAIRJ6tfAewiaH503LOP5T3CiJv6PNwCESen19JxiFGvWWUpNYg4Ob0Ufx808xPO7kEIj0Wg7yAE44if + VcsfQdv57Tpi7ZIHg+7m7pYccUfiVlp98y2wvvb4GbGuMTDURxwZ2yRsrQSxnrFA0LlR/Yqq5Pz4jgSt + 1Hr2G7ZW+RtvRfE3bD1x9wGtW3OCYBex9jMw0Ees+b4hq467v5PXy5gcaGStX3FZ2Myrh9AaiLQ9mY15 + PnZNGaglOakIp55+ibrdV42htGHPTVzL0RKehZFyYJox8tTPz/tP00Q2c4YUVU85tq9Xi8sL1db+INlO + lGub/rhoPqTZjpRva6cHN5vzdliWFduSqgYUSBzqulwLRJwbWntvcoiR2j5ZIOJs96kmdv58OmSvZJqU + qdgneboSOT+O7cEjNl/cPWzPiQ0m5hiI1FxSZKTOMRCJsWIRcwxFkjKRaV4TB+EhTyDi6UTfmGQ0JUis + dn6HuGjQpxE7sQdkcriROJfjoIhXvtFdKUffleqbXSXMrWksw2AUXeYiw2gFHifZNPdSle4eREE7smTQ + NDbqrzeM+2sosli3X9ZTj+yQpmRELH1hpy32ooNatkB0xgwyxAci6FtGleLokuN4xkXcH1biZf8WMVvT + QNSYdliOaoflG7TDclQ7LN+gHZaj2mFptJ9dakf+MstEiPoG2efrxseP6YTguhHx3yrwcMTo3o8c7v2k + UhIXUBoY6kuuFxOmU6O4t93Mnatuadw+51/1HLzqVSoFp6PWcZCR0ywgbQBl13eDgU2cMz5gHPLrWeSY + ADYPRNgI+vyJweFG8lyvB4NufUAZw6ox1Me91BOLm5uX4gRtAQPEAxG6F5TJ5o7DjbzkMGHAzZqpQWZp + SMeImxDiSq6/sHSKQ42MGvUIYk5mG2CwmHnOvdo5drXnzDQ9R9P0nJum53iankek6XkwTc+5aXoeStM6 + l/o+0wuZaScXBC1wtKRKn7nP2jFHKBLrmTuiAOIwOiNgP4R+dp5HAta2M05Wthjq41XkBguYd5nq9xUP + MZ0SXwHE4cwdwvOGeuIvtiwDjlAkfln2FUCc4+QN2X4EA05embFoyN7sNNh8i15eTBh3tznDlbc0bm+y + gytvYMAtua2axFs1GdGqyWCrJrmtmsRbNfkmrZoc2ao1J54QnztbIOTkzCIgcwjNgJp1/51I0Po34xd7 + z+ybP7NSD0k54ml2Ngb4nsgvWhoY6uPlh8Hi5kqs9SseXHmHD/qjfoHpsCOx3hhG3hXmvCUMvx98/Ctx + 0Z6B+T76i2zYO8bMN3fRd3Z5b+ti7+n2fyemngVCTnoK4u/76qMW2p3wkjTPUlJ3wmV984a8f0JPOTa9 + 828qZHJ+cZmsV2t9flDTSpHkmGRkrCTb7VXfI6PuDztKOHwN+qymN/jFnSYUb71LVvlB1GVJey0Yt4yN + lly+TbzkciDijrzLKqIIxamr5HGXHlOdH8z2BCI+rHfsKIoNm9VQqtg0W4nGxOgtA9FkxE3W8QMR1F1w + fhEVozGMiPI+Osp7LMrvF/xcb1nErOuJ6JrWlYyMFV3ThoSha3iDOxbwBCJy865jw+bIO9azDESTEZkV + vmOP3+DfsZZhRJT30VGgO3b9mKr/XbxL9mX+ev7+3QdyFM8ARNmoKxEb8T7u9gUtY6NF3cCDRuAqXuKT + 9mUwbU/9KJr7hCG+umL56gr2CcJ5KDYG+8hVFNqfaD8ot6zrUxjgU00YJz9aDPEx8qPFYB8nP1oM9nHy + A27p2w84+dFivq9rd6m+DkN89PzoMNjHyI8Og32M/EBa7/YDRn50mO1b5elPcbEi9mN6yrYxXjEF3y3V + lTuxhHSI7yHmZIcAHtqS/Q4BPe8ZovewiZNMRw4xchKs40Aj8xL9K9QbThSHnDSRd2Rsk35+3c5KrV6L + dEfKWJcNmGlPwB3U97ZzXrwrNtmAmX7FBop7y9W/uV6F2t7HVDbV2WNabZ7TipQSLuuY9z8Ft0PjsoiZ + 0RS4LGCO6tbCBiBK+0YKeczrsoD5pT2dPCaAr7Dj7NJK/TnvilWS5g9lldWPpJzAHHAk5uIHAEf8rCUP + Pu3YN6TtxNXXXf4Djf/g8c1ojihpGNu0V79UROU3bICiMPPag0E3K59d1jZX64vkt3fUhrmnfBtDBXh+ + ozmcskctN36ZaeYRts1GoN0eYutKv9hw2G6zF6oaFXkxLy5+I8oV4Vto1SZUS3ZPft4oBUIqL+77S2oa + KMKzfKDN/LUEZEnoqdlRtk1PSukZqua1gF1KuklcFjZ39ZNeNlBtOHpLAMdoPzt+Ux72egNSwYqGqLC4 + zaGujHfdYIMR5a/l9PZ6et1s8vR9MfljSlsvD+NBP2HJAAQH3ZS1myDd2z/P7hekF9RPAOBICFvoWJDv + OuQioYx8XM4x/jqI6rVv1ZvzeA+SJIcVTpzmOOJ1eSgIT5I90HFKUT1la/0izCZbp3VZJelWfStZp+MH + x4OiwZgrsdXHIr9BUMPkRH0SlSScV2syvemP6e10PrlJbiffpgvSbe6TmHX8ze1ymJFwS3sg7KS8hedy + iJGwv4zLIUZu9gRyp31xptQH9d4SKpCAIhTnKc0PETEaHPHzChlaxrhFLFDCmuXXLGdDIlZ5SvyCm3+2 + IhSHn38ykH+L75+W8ymveJssbqYXjp7ErYwiYqC998vX69GnEOnv2qTe8j4tNhRBh3ieukrXNVHUMIbp + 2+RqtEF91yY5O3y6HGYcXxu7HGQk7OxpQYiLsMTV5QAj5UayIMCl55vH73vgYICPsvzbggAX4QY0GcBE + 2s/SphwbaTl1TziWGTWVZn4KEZdOm4xjoi2YNhDHQ3n34wQYjvlioV/JT8ffySfCsYiCamkIx3LcZpsy + AemBjpM/hY3gjp87cQrCrrvMX9+rm1WNMmqa1wBB5+6QM4SK6m2zxeK7+mpyPVssk/u72e2SVE8ieNA/ + /h4G4aCbUPfBdG//+uPTdE67sQzE9ZBuLQMBPbqDobulufpnXREa3ZDDjcS5jX0yZI38GUGVGzfiGRsq + QGOQqxGMdyOwnx0hOOJnXj9eD3aft59sq3JHfRUYFfQxvl2PfhygvmpxtO7JCbAdlM7J8fu2YVmpnvq2 + rHYUzQmyXbTOSU+Ylg/j8Q8WR03PD356fiCm5wcvPT9w0vMDnJ4fyOn5wU/P6fLL3TXlddqe8CyHgu5p + mN7UTEBc3d0ulvOJavwWyfpRjD/wEqYDdkqvAoQD7vEFBUADXkJvAmINs/rkMy0JToRraXYNFuuaMMnt + gaCzrghPzFzONebl+EP1egKyJKuspJs05doo2XkEDMd0ubia3E+Txf1XNQgjZaaPol5CWXZB1En54R4J + W2fJ6uNvuqtLeOyH8aEI7W4R/Agtj0XgZuIskIez5q5QXRVC/wnjsQi8QjJDy8iMW0RmoRIiI9NBDqYD + ZWMPn8SstE0qINYw3y1nV1P1VVpZsyjIRigBBgOZKDlvQr3r7tN/J+uVvCCsBTYQx0OblDYQx7OjOXYu + Tzr+qSdsy4b2Szbur1D/sdFFNdvoRQOS4nJQ1Lt6jVF3tG1vnkqqzm9KkZ4gz6U6rpvxnV0Lsl056UDy + nnAsBbWgt4RtUX+4WK9WFE2H+J68oGrywrcQVtwbiO+R5KuRztUoLTWJO8T31C811aMQ2yPJOS6BHFda + qqZDfA8xrzrE8NxPb/WX9L4oaZ73K5Jksi6L8fdaWAPEk81De3qAjvONegVQuab6Wgqw0R6yOhjiI7QB + Ngb7KlJPwicBq8qr7IFsbCjAtj+ohqE5XZms7FHfy/nV8O/V84cvG9V+1XTfkfStutHJ0vcXhHl+AAW8 + uzrbkX95S2E2dcf+m2fUJGrdZNstU6tR3/uYysf3F1RlS/m2LomTe6rwBAJO/Wi42VS7JFt7FPDKNC8O + O7KzxWDf/jHl+BQG+Vg3UIdBPrlP14LuazDI98K8QOz+zh+TjchFTb7GEwg7y6blrB442iMLmjkVZoeB + vkw1cVXNMLYg6CQMPm0Kth12apArxm9fC7GguRJ1lYknTnoe0aCX8rANwQF/Mw96yPI6K7p17fSUARx+ + pB2rF7ZDemHt30lrogAU8Irdht4paSnfVpTMjtMJ9J37UmYvSV0mNbnmN1DfWwlWBnWY75NirQ/t4XdH + PQEag1e0LBhw/1RVstiTFixCLGLmtBInMOBMsi1bq9iQeT9+NxQQht30u62lQJuedmLoNAb7OOX2J1Za + fzLbxxMIO2UiSS/OQSxoZrS8LYXZSBttACjspXeBWwq07UtOeVQUZmsKA2E1KUzD9oN85GgVBvoIK3lt + CrM1B2NtD8Wapz3hsP8x27KuV3OwsWTdmxoDfaSXPlwONP4tqpIh1Bjgq6t1qlrBHb3En0jQyqnTGwq0 + 6aE6Q6cx0Jev05rh0xjiY3QQWgz0FfxMKUK5UvCypcDypSAcIulgvk9P8DyQ6/GWAmw73ctturtkZY8C + 3jIvnwW5F9Rhvu+JO9n9hM92nz5SfYZ2vStbfjL4Uf5mdbn/dvvayy/TOfkFTZuCbIRBocFAJkoXyIQM + 114U8AOQ0WLUgEdpt/xih+hw3N/utMD2d7jvJ76a7WCoj9RJ9NHeez/9lkwWt+fNi/RjjRaEuChL2DwQ + cD6rEiLIwobCbKxLPJG29a8P735PZref78gJaZMhK/V6fdq2r15rIVlmm7St6j+bZ42rdPzKWpdzjGXy + qEKNb6csyHbpx05655Or2b2q3ZrUoVgB3PZTc9/P8yZVr7/QziTzQMi5mNy3LxB8HT/xCtOwPbn//olw + vBeAwl5uUhxJwDq9ikgKEwbd3IQ4kYD1/uvV4p9kY0MhtkuW7RKzqa/P/my2y6HeVJgDisRLWDxV+aUg + WAbmUffafOBe0583rwVx5UcYdnNTeR66j3VjRDZqCHElk+9/sXwaxJxX8xueU4GYcz79F8+pQMBJbKnh + Nvr4V347Y8KYO+oe8Ax4FG55tXHcH5NEgTZIfx7VDrkCNEZMAoXaJP05r106kQHrJdt6GbJGtlOIB4vI + T/hwqseVmsEyM4++d+cj7t2odswV4DFicmE+VD+w2rUjGHCy2jcTDrk57ZwJh9yc9s6EbTd52A+M+Nsh + O6eps0nQyr1RABzxM4qvyyJmdoLArVr7IbdJ82nYzk4OpCVrPyQ3YwaG+S55vkvUF5OwjmBEjISwcj8o + QWPxm2JUAsZiFphAaYnJiGAezOPqk/lQfcJtcn0asbNTex6srajNbE9hNmoDa5Ooldi02iRqJTaqNhmy + JrfT/8M3axqyEwepyJz66c8RbTc+TjU+j7vnBkaq1pfYd0dorGp9IyqhQu16zHAVNuBRopIp2M6zhqwO + GvJe8r2XQW9swo9o/4Gv8foAiCgYM7YvMGpcbnw1ooANlK7YjBrMo3l8fTUfU1/F9RXC43PrO1G5MR+s + FXl9B3iMbn/G60Pgo3Tnc1ZfAh+nO5+z+hQDI3Xrc17fwjUYUdTtfX6R3H+a6nUXo80W5dlomx5YkOei + LPoxEM+jnzLrDf7SYpOsRTV+WQrGexGabeuI1obxTO3mH5RDWzzQcSbf/vh8TpI1hG35oDL86/Xni4Sy + DbUHBpzJ4svknC1uaNe+X4kLvT2Qfj2S9CYQgoN+UUT5Tdz2/zNZHYpNLnS9QyqwFog4dSnOtvogDMFz + mwIkRpU+x8dxJW4sahXxT6CG+Gdzg9OT+UhBNl3/8oxHErPykxQyQFHiIgzZ44oFZHCjUHZ06gnXUr/u + hX7/hbIJjU+i1maBI9PbsJi5q1HEhic/4bj/SeTlnu/vcMyv84Irb9mweVJspnE/wffYEZ0hE7mOgvhw + BFrT49NhO2GNM4K7/q5VpVk7yHV1BZbm6iDXddw9+XQTcPZJHqFy47a7Hr9B1IDIiHl3M7v6QS+aNgb6 + CAXRhEAXpdhZlGv71/fJDfPXWijqpf5qA0Sd5F9vkq6VvYsuggf91NRA99IFPianCr6fbvf5t8n9vSbp + l22QmJWT1iaKerkXG7pWetoaZG+dT26vk+4dibE+k3FM6i8ifSWJWsTxEGY4jt93DM0ifZKjISBLezSt + Ph1U76SsD/cmdDIHNE484vZhJuOYNplMV2pIti2rn8mhkOlWqFHadisoez4Pm5yo4oGWb+r7rqF4o8sO + iZyY24x4bqhNObZ20FNskp2oH0taejgsYJavsha746EX+ucl64Osm/MRiCk0rHPiN1vD6J9NCnOiHNu+ + HL97wAlwHVIcNiXjZjdBxymFoGWaBjwHvwzIYBmgnUFrIIbnavS5GeqrFtdcHKGfayCGx3z8QtkyxANt + 5/FZC1Vpcpbx/ybn7y5+05sg6ZMCk/Tp5YLgBWjLntwvFsn9ZD75RuvlASjqHd/z8EDUSeh5+KRt1S+Q + 7n+u5bmqbQTh8HiItc2rbPxzg+P3HUOuDx8uHpLx7686mO1rjstQ9eCedF09Bdkod6IJ2S7i+N5AXM82 + PeQ1tc7zSNtKnDEwENuzzdMHUtI3gOMg3qb+vekcYUWROWjASy1kHuy663fJuqoT2uoaAAW8G7JuA1l2 + +3O6SEGg6xfH9QtyCbJIAJZtuq7Lip7wHQcYs1+7PVmnIcBFrISODGAqyJ4CsNB/GPSr9lJyy3uPAt5f + ZN0vz6LuftoY1MZAn96US7Vc1CrJZm1zJpNyn/46kG6CE2S7Ik7zQ3DETz4JD6ZtO7HL5PWTdALTW9We + wmx6Z0rBUzao72Xmj4MGvUmeVg+Cft2AIhxHb9tZ1TFhWsNgFBEZA/odrHJskyErOxM8gx1lr+fHVO9Z + 9+7b1S13k+l9snvYktrkgGYonh6vxIc7WoaiNU8pI2O1DjxSURaCG0GzsLkdTLxBHoGi4Zj8lPMtbjTm + masgDLpZdyd+2mrzqd7ki6TTgOdoLpsxInRQ2MsYyzko7G3GLfqMWNpEIGrAo9RlXIy6BCO0ecpJdosE + rZxEt0jQGpHkkACNwUpwH7f9kj+ilaERrWSO1iQ6WpOMEZYER1iSN26Q2LiBsm7r+H3f0AyWqC2HBQLO + Kn0m6xTjmv4WNMvfTkupil1Nn3bqKdt22FNOEu4J20I76bAnIEtEhwkUgDE45cNBQS+xjPRUb6OsgbZX + POt/0Y7M7gnHQjk0+wQ4DvKx2Tbl2GgHZxuI5bm4+I2gUN92aXL6nhjPREzjI+J5yCnTQ7brw0eK5MNH + l6anzZHxTNS06RDPwymDFocbP+Xl+qfkelvas9Pz8gRZrveXlHKuvu3S5Lw8MZ6JmJdHxPOQ06aHLNeH + 8wuCRH3bpRPandIRkIWcyhYHGompbWKgj5zqNug5Ob8Y/rWMXwr+Sk4dYXGekZVmXnrN7r9MFl8SQot1 + IgzL/eTr9CK5Wv5FeszoYKCPMP1sU57t9KRwJx+IShP1vPuqXAvdXSNrDdKwkpYhuisQ239TN6+2qd62 + nH9fLJPl3dfpbXJ1M5veLpuJNcKYDjcEo6zEQ1bo8/IOaTH+nL1BESFmUqrUSHYqe9KHt7sAyzriaiqx + Ebt9TcjKEapgXPX3TD6+RdI7pjFR3+Tneq5wZEJ9heBBP6H+gumgXc9wyKqKvCMNCxxttlh8n85j7n3b + EIzCzREDD/p1gYwJ0PDBCMw87+mgXRdssYsI0ApGxIiuA3FbMLoujztRp3riLrLAuarBuBF3k2+Boym2 + /Q9uSbcEcIyNWJeb/lnOMQk40RAVFld9zXgkIcW6Gn+W17AJjipe9urbO1HUydM5J5glGI6hum67VWyc + RjIm1lO5r7bx0RoNHI9bEPHyZy7L45hNHo7ArGTR2nUvdd5zM7ang3Z2Vpp8H+H7Yjq/vVvOrmjHFjkY + 6Bs/6rUg0EXIKpvqbX9dfPhwPnovoPbbLq3L0j7NKprlSHm27kldUzl1lSPRDBiMKB/e/f7n+2T611Jv + 0tAuaNAn8Y6OgfBgBL1jT0wEiwcjEN6KsynMlqR5lkqes2VRMzcVBlOg/TSRP2PkCgf9m4uMoVUUaKPU + Jw4G+h7G9wJsCrNRNrjzSdCaXXCMigJt3FKEl6A2+3m/+8SCZtICHJfDjcl2z5Uq1PN2J+21nUHKLAHG + exHUTXbOKAZHDPLpV9iKTVrpN6lqUegJNknXQxYwGumkV5fDjcmqLHOutoEDbnrZs1jPrMN1+VxT3r1F + cM/f3EqMCvLEecY+U1m3oot7fl3r0duHjgJtvDvQIEEru6zZcMBNT1yL9cztwsY8k1RtD3rO5sDp+oUo + 7CjQxmmLTpxtTCY3f9zNE8KxwDYF2ghvvdoUaKPemgYG+vSrLAyfxkBfVjNsWQ26CGMrmwJtkvdLJfZL + m+m3Dc+oQNe5XM5nn74vp6omPRTERLRZ3EzaVRSEB9zJ6jW5nV1HhegcIyLdffrv6EjKMSJS/VJHR1IO + NBK5jjBJ1EqvKywU9bZvVhKmXDE+HKFc/Vs1pzExWkM4in7TICaG5tEIGffyM/yqybWiSaJWVSmdx+Tp + iQ9HiMpTw+BEuZrOl3rjanqRt0jMSsxGg8OM1Ew0QcxJ7l07qOud3X5mpOeRgmzUdGwZyEROvw5yXfMb + +u6SPolZqb+35zAj+XcbIOBUY813SSWeyp9iQ/aaMOw+16M36pyDB8Nu/SlHqznASO3zdwxg2ohc6Bej + GJfXo5CXtNmtg0G+A/0X+70N/VfWzYPcN02bqnpLemtistOEA24pqizN2fYWx/y8mTCIxyLkqaxpCyQx + HotQqIuIidDzWAT9bk9aHypmgBMO+5P59M+7r9NrjvzIImbObd1xuJEzbPLxsJ86WPLxsH9dZXW25t1W + riMQiT469uiAnTiP6LKIuVlVVbHELYp44yqCwXogshoYrAX6u5j63Ac2IFGI64UhFjAzunZgr26X1utH + sqqhABunewj3DBmDiSOF2YhPzCwQcDajwYhbwOGxCBE3gcNjEfpCnOYPJS+K7RiORH6UhkrgWF3FRdq9 + FeORCNz7Wgbva8rr0xaEuKgPOywQcpaMfrGGABft1WUHA3y0l5gdzPFN/1pObxezu9sFtaq1SMwaMV+N + OEZEonbBEAcaiTqis0jUSh7d2SjqbY654XQaYUUwDnli08eDfsa0JiRAY3BvgdAdQO0rWCRqlfG5Ksfk + qozLVTmUqzI2VyWWq7z5Rmyu8ebu7uv3+2Zia5PRxhg2CnvXdZVzpJqDjZR9yl0OMVLT0uBg42MqH7nJ + eWRhM3mrdhB23M3ar+ntcj6bkltLh8XMPyIaTEwyJha1ycQkY2JRH/JiEjwWtYG2UdxLvgMcFjezGk+A + D0dgVLSgAY+Sse2he4LahNoo7pWCfblS1EFvVG7KwdyU0bkpg7k5u11O57eTG1aGGjDkbh4OFXX1Sjef + 0KCXXXm6hsEorGrTNQxGYVWYrgGKQn0Yd4Qg1/GZGi9jTRq00x/KGRxo5LQRSOvQpjN9ytyFITevzcFa + m3ZJEHGS3CIRKzfjTyjmbTbWZt/RrmEwCuuOdg1YlJr5DAoSDMVg/5AafRLVfEX3u+liTWG2pMw3PKMm + ISun0YLbKlbPA+lzlIXIs4JxM3cg5KQ/Pugx1Ec4mMMnQ1bqkwkXhtysPpzfe1OlfXpFf2XN5HCjfmuj + VrWc5KpPAjhGUzfrP3D8Jxh109duOixspt5bPeb47r9/0uf3kvPO4GAj8YVDA0N975jCd7ix3YqX623p + kJ28WXdAAcfJWMmcIalMLVc9BvskrxRIrBTIqDyTeJ7N7+8WU04h60Hc2azIIj9mhASBGMTlCTYa8NbV + QdZsdUM7dv22Om+G2SIxK/GOMDjMSL0rTBBwNgtH07quyNITGbJyesmQYCgGtZcMCYZiUIfvkACOwV0E + 6eODfvLSIVgBxGmPo2AcN4EbgCjdBAOrxBosZKZPTfQY5CNOTHQMYDolPSvzLBqwsyo+pM479hI4uW+w + mJm3CtbHYf95InZplnPcHQp7eYX1CAac3MrV4QcicKpWhw9FoM+2+Tjij6hVbRzx8wt6sJxHrPMEDViU + Q/PUgL7kDBIgMThrzhwWMDM6VWB/itOVgntR9OmbE4XZqJM3Jog6t3umcwu1S7GrMRHHcCT6akxMAsfi + 3tkydGfL2HtODt9zMuKek8F7jrzO8wghLvI6TxMEnIy1lD3m+Zo3Wvhv5EECPAb5HRmHRczM9+p8HPOT + +7cnDjEyeqI9iDhj3jFDHKFI+vXOdar3tLmmroAPeEIR27frbg+7laj48UwLHo1dmOA3upxPed1ZSDEc + h96phRTDcVhLOwOegYiczjRgGIhCfesL4JEIGe/iM+yK6T28E4cYdSv5Bje5rwnEi77FXYkTazH7g173 + HiHARZ65PkKwa8dx7QAXsXS1COChlqqOcU3Lu/m0OaFknYu0ILamHo3a6Tlroai3aTfIr50D/ECExzQr + okJowUCMQ1XpnbHXxMXbuCYcj/7QCBIMxmiuhdjNRi3haLIuKxETqBGEY6iGST/AIe68gUlCsc6bcin5 + cTrBQIy4kn0+XLLPdVGM+xmKD0dgvKwNGkJRmkeOB/oyWUwSjBWZLcO50tcTUZWnpQnGE1VVRuRQyw9H + UEPGff0YG6e1hKO90Fdlg4ahKKrRbtcDxoU6adB4WZFxS0JWZHjuk3sqJolau7Oj2TXLiQ9HiGkl5XAr + 2Xylawz0lsrrnzGxLFEoZlT9Igfrl+aVA7FND3kdEaMzDETh3+0nPhghpt6Sg/WWjK5J5IiaRH+HdHY2 + xgcj7A/VvpQiIkZnCEaps11MCI0P+hN1FdlLZJRWEo5FXkkE8MEI3VHb61VElJMDjfQWFdhw3aVnmpm9 + lSOKe1mDro5ErXlZ/mQNqXsYdDNH0+hI2th3lVNFmDju57akA2PNh35/Uea1nwevvXl/N+/myDgRbAEY + g9dDwnpHzSNGbmr3MObuVkjx7hiLRyN0Lb+6jvpRMqNYjkAkXv8h3HeIaW/Dba3+tN1Ag5v6HY3a+a34 + UAse0+KFW7vYlm64lWPsumOCjvPPCWP/zSMEuIjjtj+ht2n1H6n1UMe4pul89vlHcj+ZT761+83uyzxb + 056LY5KBWOfJY0ksYLAiFEdPdleMGxyThGLRi4lLh+wPrCoQVgzFiUyvB6RetL6UFY/qNo7I/04QisHo + 1AF8KAL5NnTgkFu373y5pofsjAWsiGMwUty9flIMxsn2kVGy/YgYSSrX0XG0ZDBWU5VmQkZGO2oG4sXW + MHJMDSPjaxg5pobRX9Jl5g1inTRD8ThdMkwyFIs8vQIaxkRhTLIEPIMRyR1PWOHEYa/OC6zKaz6qRLPE + krEti49D/ubHsPUm7dvJK7TgNYTNmaj0dRw9BvrIDWCPOb5mDpwzMjBBz6nHxulP4pL7HgN965RhW6eg + i966GxxoJLfiPQb6iK31EUJc5FbZBGGnftTMyd8WBJ3cN96G3nbrPmc0QBYJWulVssG5RuLmQ/6+Q+ov + p4fZ5EbQhQE3yxlwMZpPG3W8zJXa6AptxpuM4FuM1BXe/srupuahD6R7zPGp/9rodRzdbtep+hfjcBLU + gkTjLD1xWNdMTREgLZrJ+fRQP5Zq1PzKWYcDGsJRVDVFfbkfNISjMPIUNEBRmO8ChN8BaE9xKevJtubk + wZFErJ/Elrq6zkYhL+MVJ/wNXeOTZJXVsq644g6H/Oxl0ENvOES8Wxx8r7j9sHtji3vn2DwUoV5JfQlp + /kC39yxkPmQbxl2iKd/GmZxC36xuHx2u5Z6u05RvS4ytWahOkwXMx6dh+iF4klYiJfs9w1AU6lbMkGBE + jEQUT9FxtGQoFnkDaNAwJkr8TzpaAtGOff6YbDIcQCTOuiZ8XWTUasiBNZCct8rgt8ki3iILvj0W8dZY + 8G2x2LfEht8O478VFnobjPsWGP7212mzhY3YNO3cQaYPgiN3FFicZjcU+jQywAMRuCf5PARP8dGf8pMm + lCLcbmug18rvtIb6rM16klwUZGfHQUZWJxjtA0d1UQd6qBG7ggztCBK1G8jATiDcXUDwHUD0y33sQrsL + lNodv9ju8HK7a6Z90s2/ac4T5vgyqTeuyDbdcwBiSfBoz36qf8jzeg4bMJO3HnbhATd5I2JI4MagNaDe + OgZVX6hkJz9R6THQR36i0mOOr1kq2XRg11VO73D7OOqPcKNe/iXDV0tdBuKv/NinlRTJtip3yeqw3RJr + Ko927c2CrHZSniY2QNdJ3sMI2r+ItXcRsm8Rd7tpfKdp1i5IyA5I3XwVY7LdIh1r9/S4WaJGkpqg42zP + 1eS0mBaJWBktpo1C3ohdpYZ3lIreTWrETlLct4vwd4piTgkNnxAquaMAiY8CJHsUIAOjAObeXOi+XFG7 + awzsqhG139fAXl/cfb7wPb7I+3sBe3ux9vVC9vTq767NgdgRtVHUS2/vHNY1G9lF7jy7cMhN7j579JCd + 3IEGDV6U/b6s9HtmpzkUYgyPdyKwRlrIOOv4Z2pXxuBcYzPkojfsBucYGeufwJVPjL3zwH3zju9xUF8U + NDjc2O0OIGt16z1w9ZbEjvX0nrN+rqc8G29VhwV6TsZseU9hNsaMuQeH3MRZcw8OuTkz57ABjUKePXfZ + 3pxeZMnsXgnm08VirNKCEFdye8XSKc4wrrKkViOSZKUGxofiWa9gqcVOVbrp+BPBgpJwrOeqLB5U9fSQ + SUJHdNgERF3n5Ur12JLq/B05jsEGzecR5vOg+SLCfBE0v48wvw+af4sw/xY0f4gwfwiZL/niy5D3d773 + 95A3feGL05eQebXnm1f7oDnimlfBa15HmNdB8ybjmzdZ0BxxzZvgNcuIa5aha37Z7fhVqIbD7vMY9/mA + O+rCz4euPO7Sh679Isp+MWB/H2V/P2D/Lcr+24D9Q5T9Q9gelewDqR6V6ANpHpXkAykeleAD6f0xxv0x + 7P5njPufYfdljPsy7P49xg31IJrDVFS3uX0vfpNVYl0fV7iQY4VkQOzmDdO4iL4CiFNX6U4//Bp/biuA + At5uxFGJ+lAVZLVF43ZZp+OnVEA45C73fHVp9u6EPL+4fFjvZPaUqH8kP0cvrwLQoDcRxTp5OY/QdwYk + ykasWW7FIUaxXjUhV3k5/pEtbsCiqM938iF5+Y0X4oQP+S/j/JeI/+dmyxIrzjJefPjILYcuGvTSyyFi + QKLQyqHFIUZuOUQMWBROOYTwIf9lnP8S8dPKocVZxmRdV037RHhi6WC27/E5Wa/W+gdUr/uaorRJ31pX + 7y+On7Z5K6l6QOHFUSWTceUd5dm6ssgwGqRv5RkRW7uHRpsoxGLg06D9mOQ8u0Hb9qLklzaXhcyRJQ6V + ALEYpc7kACM3TfD0iCgnEI9EYJYViLcidBXgY52ucvGRtKE1TOP2KPmQW3X0X5/GP0/CeChC91HyWFYF + 4fkGwlsRiixRX2IUcxuEnPSCboOGUxbn+vXO7vFrkoviYfzmRDDt2Ddlkm5WJGWLOB7dQaC8o21BgItU + Yk0IcFWCdNiGywFGmT7RdRryXeVG5w1pkQOAOt4Hocp7mmd/i02zvKIuk/GHAuEGL4reH7XM1kJVdLlY + 12VFjOHxQIRtJvJNsq/p7hMJWLt7oq2CtmXVjNIJ6yQGRU7MTLZLoPTXSDFM0HFWYts8LteVUTOD1Mw0 + /C2qkhQB12DxdLNWFoIXpYMdt4wsS3KwLNWve0E9OMoDIadsT+OpqKXHhSF3s1A2SVUZKFUZEBU9gGtw + ohzqNbOGsMjeuhLikOzKjaqM9bpJfQEVZTsZjDciZGU3VypV55V66gFM23b1p6JM5GN5yJupxvGLOWDa + tuvdltRdppfm6cTrLkP/Kd1sSL8jbLKj6g/pKdVTvk2vOlb/TdV1GOjjJjmAG/4iSfWmDYdVsi4LWZNK + I8Da5s0meS6r8bs+mIxtkrJ9Y6eWquwnq9dakKQAbvlX2YPqNGyytNBlhXrNAG3Z1+X+lSztIcu1UV13 + Tk5ZnGUUL3t1VxBULWA5jilL/ZEWZxv120q7sqgfyp2oXhO5S/OcYoZ4K8JDWj+K6gPB2RGWRV18lRYP + gvzTbdB2ynZoou5astVBXW8l8rTOnkT+qntOpBIE0Jb93+m6XGUEYQtYjlyN9Dil2+Jso5AyqR/VrWkU + hjlFDQqQGNTsckjLusvyvFlMtcoK0pAPYgNm1e9pTrRg648CJ0aRqVsuec4240flLmcby017TgujfHgs + aKbmnsV5RlVNNkWGXHX5sOfu+n/v2tuQHwb1YBHZqe/xaARqveSxqFmKdSXqqACmwouTy8dsq4+5ZKaR + xyMRIgME/LtDHtPoYgovDre/6bGgmXMfnzjPeDj/yL5Wi3XM7UG41FE3gMJeaothcrBRdyrmc2ZaIA4/ + UvGO6i3e2ZZD/ttL8wlFdIJcF69lMDnPuC53q/Q3oq6FYNclx3UJuBg5a3KekZ4LcB40+UzvsLso7NVP + ozhSzXlGcpV5ZDwTp8yB5e2FdTu8QPdDqcp00byerIcD5eopKw9SjQZUgdJbEdeUkjPosiMXzWxa37JQ + IrmsZd6Xz7RS1QKWo9LzSrxxoIv63q7P0XyHKjZZ2yw2h7VQSbMmOXsKs+mB7T5PudoT7vhl9jcjbQ3M + 9nU9LbLQ5ADjMb2bf5C9Fg3ZeZcLXK1cp3VNK/VHxPY0jxPI12Vijq9mjxw91jPLWo1T14yrtVHPyxEC + pl/Vpe5+qUQuUkoTYoOAk1j595Drovdcegh2XXJcl4CL3nOxOM9IbcdPjGcil44j45pe2MXjBS0fjNES + PFKy2ldy6gG0ZT9wJ34O+KzPgTsIPeAj0GfyZPozMJvepK5Ok/7BAsXo04a91E9Tpcx1Hbxtn2Y/7tK1 + anPSiw+j348Z0ITjxYcaGeXD+PfacEMfZX2RJZPF7XnyabZMFkutGKsHUMA7u11O/5jOydKOA4x3n/57 + erUkC1vM8D2m6n8XzdGdr+fv331Iyv34nVNhOmSXYnwNB9OGXS8bK5s1ZOtcj5FEoZeLjL5HMb6PsNHJ + dnWlN0C4ni6u5rP75ezudqwfph07r9RtQqWu//DbPVd7JCHr3d3NdHJLd7YcYJzefv82nU+W02uytEcB + 7x/TW/XZzez/Tq+Xs29Tstzh8QjMVLZowD6bfGCaTyRkpdVFG7QuOn1y+/3mhqzTEOCi1WsbrF7rP7ha + Ttl3lwkD7nv19+Xk0w29ZJ3IkJV50Q4PRFhM//V9ens1TSa3P8h6EwbdS6Z2iRiXH8+ZKXEiISunQkBq + geWPe4ZLQYDr++3sz+l8wa5THB6KsLxi/fiOA42fL7mXe0IB75+zxYx/H1i0Y/++/KLA5Q9VqX2+6xpp + UgBIgMX4Ov0xu+bZG9TxHuryvj3U4+v4dzN80rZ+mixmV8nV3a1KromqP0ip4cG2+2o6X84+z65UK31/ + dzO7mk1JdgB3/POb5Hq2WCb3d9Qrd1Dbe/1ln1bpTlKERwY2JYSFgy7nGGdz1d7dzX/Qbw4Hdb2L+5vJ + j+X0ryXNecI8X5e4RF1HYbbkdkKrwhzU8S4mvFvKAgNOcsa7cMg9fptqiPXNh1WerRkJceQ8I/G8LJvC + bIwkNUjUSk7MHvSdi9kfVJtCPA+jGjpCtmt6xbiqE+S67nUEUYtK0nQ95xlZN6HJ4UZqeXHZgJlWZhzU + 9TJulhOEuOg/Hb1T+o+oPxq7T1STMb29nl7rvk7yfTH5g1St+7Rt74bY5ObC5HDjgqt0ehqzxeK7Ipit + pU/b9tvpcnE1uZ8mi/uvkyuK2SZx64wrnTnOu+VMdfemn0m+I2S77r9eLcbPEvcEZKHeQD0F2mi3zgny + Xf+kev4JODg/7p/wb7vkV7cAHvbTE/EyUO82n+upkz+bmkSP6sh6Gx/0s1LIVwzHYaSUZ4CisK4fuWLO + NXpXpUeHP8hZd6Ig27++T254xiPpWMmNO9Sy85p1rE1nNehIa87rwWH9t4jqJFSTsCuRQP3BGTQhI6Y5 + dzQ6x0ej85jR6Dw8Gp1HjEbnwdHonDkanaOjUfMTTjKYbMBMTwQD9bzJ/WKR3E/mk28LotYgASu5Lpoj + o/I5e1Q+D4zK59xR+RwflX9fTOdth5Ei7Cnbpnfxp3j0931DMrn5425O9bQUZFsu57NP35dTuvFIQtbv + f9F93/8CTHo+l6U7gpBTtbR0n4Ig1/yGrprfwCZyT9ICESfxHjM5xEi7vwwM8DVD8gVxnYRNhqwLvnYB + eKkTAycIcNErVAMDfPPpv/5fa+fX5KiNRfH3/Sb7Nk1PZ5LHbG0lldrUZsuTmlcKG2xTxsAg3H/m068k + bIOkewXn4reuhvM7IJAQsjiCYVpDk2R34k3IMCV34lXHEAV34iAjed/++g82qWSqI4jg0OlNQ5C+/Yq3 + MlpDkCTXgC5/Qdk75X4c1gtNrz9a7bPl611SWpfcnNtLX9gVytssN8u3m7iR25Q+xCdOmriqLLWZL+di + +XR1R+SyhhMEAvUc0cgqdunvv10/ItbHv5TmyWhevq0kPC2jefuiKs7mm2cJ9S6OsYfFb5HYkBgj5nS+ + VHILLY6xh+9k5PhBH3NQ3zs5XotjbDMled0VuBFoF/Platp2ham6Eo+pnnYQXlv2qprppNtMFUKo1cbI + /e4oR2sxz15RzBN5hG/fdNedwpQRONWl6s3qhbsmL8y3TVXWmeQU9ObkMIGfKs9tZRfjTN/1w6Xp8rLO + evTKMxTObWXbx1DibsJaTjI4p0PXXNohIvHSvQoL0YPEvdQjvNScl02Z6GUWg5YlqzQzLdzeNHIfQgeH + EXFq6jVlNQFwHjauzyZkySxGfdwByVDg9HEHc0vou33dhSFRUV+VFt8vWbXC7kpwXLK9+eua65TVsAep + pxyG70dx8qCjiLrgbrY4diJ22ehrwVTjkLblob7YdtE2kADPUzLU4cklwg5Sh7viIRd9st3eyd7+++tv + CHMic3jDwwZ7ObprCBJ6v09UBE302I4+q4eNdXGAgVpDkXQ7baJw03OmTjhzqiboQIjuVEOQ4OZiKqN4 + ly0Ou2wJ0vCVpq5JMO+uZKii+4bsd5ke0rRKmrxcFM8yZp3glomHOF52WXl9vrafkbbJy0/p+zm/flma + KvV2ATznYTHv558/33Y3f67zJmALvV+eErt7mnfZvv/05SHH4EPJY7m+N3nHLvCnQUs9zbHKzz0OdI5B + OFDBjk/cO0z6MIYuCUANxTNs+KWcQzg+rRloBftKd41Lsr1h07qYVR0QnCMkmPaxeqlN+XeFUkUOwwMC + 4WKGLiSD1iyA8YBbVl8a5aLjWqR+zgG7D2lA3AOvpRxixseOVa2ysYQlLusLjh1Zu72Jgv2tqYzk9beG + Y3yuKwGfwhB+gv6TK3SZw/UXlIojdJgm26uxXWjbg4arMql3HK5XGns5GkUUy77ooEseMHKKL3phCrQs + GY+eYwGUR1m/flrl4QFIDwWtgBIIKaab94qjXT3lgL2wjiKKBf+C5ugoIlytHR1JhF4vRxHFEjRlnpKh + rrnkTBYjs4O5seWtBotyfYexU5Xtr8ObiJGvdcnDmOn6Sh7jRBwfUpTLiNOjMJMS8iZ9Lbpy/yHszvIM + 30mVhzp9K/ujeaLthqWmTnXzVqdZrd6KTmC8CDk9juG3wB/mhT97fU/uGYfAuySLYHzQhF1SzLChRtfV + MUTd41p3xFNAxMPk563yuAEYj6GrB3WMKPUcHX6Tj0CiXnlzAdZdYwGMx+0efhEZ3NUz9C+r6Fz9WnUn + EXdRnry8PP0i+FnIF4ZMfPjEF47MfZldf6e+2ubvyMwXRh7nK925X74KJU/wXOxQrOT4p0KOCcyVCoQj + 0wTLHewgom7zl/IcEcWyUXU4zcooHpKR7qoomlKqeMZxVubx9PH2cMndRBQLL7lRRvHgkrurKBpecqPM + 5dnRZLDgbhqCBBfbqCJoaKHdRQQLLrJRNdKOp3yPN7KuaqSVSSbNNCSkBBdM7/N1BBFL3PNkBA9LJPJk + U95Omo5JSAkuXJI7tiTzFSmhtNqjS8shj5VDLkwJDZUUFUsJ9XUEUVKj8liNylelhHJ63kFYykxK6H07 + nBIaKikqWjvyWO1AU0IdEcFC26yca7NyeUooKSbYcEpoqIxRhQfNpoTe95CkhJJikv23EPs3Q4RTQkMl + RZU0CEwrgKSEOiKCJUwJ5fSUA5YS6utIIpoSSkgJrigllFZ79DUpoSyA84BSQgmpyxXneZJil70iz5OR + e3xZnichdblonudUQ5OQby99nUeU5XkSUp8L53l6soAHJpS5Ko4GfYdNSD2uJEElEEaY8IXnE1TCzcs/ + w6W0IRlNUPF1ARH80N1VcTRBkZLJId42uDCp5JDbJuDz74kk4AiaoTDP0/wbzvN0RD4Lz/P0dQFRVAnp + PE9/C3q/8HmewVbsnmHzPIeNgspC5Hk6/8ZPna0pkjxPX+cRxXmetNqlS/I8fR1P/CpFej0NeZ4nrXbp + sjzPUMlT/5BC//CYaJ6nI3JZWJ7nqKAoaAWi8jwn/8eqDpHnefv3F5TzhWBITu4LfW6TxMw/6n0jIROI + eR+8QENC1GXlmcyexbozmD36uszXnsEVMe+z7kwGAuEiy1pl5LN8UWnFsla5nQSlFclaHfcRHT9zxJJj + DI4Kzlp1VRQNzVoNlR4V7nhRvS5Zl4vrb4k6W0xPS9a75vrWKxrHWLsobhIjraHkhZZ5m91IRwo2/EjB + Zs1IwSY+UrBZMVKwiY4UbIQjBRt2pECatUppI2S8EMis1etGQdZqqCSocFu0YUZMNuIRk01kxGQjHTHZ + 8CMmeNaqq3JpSNbqbf+QgGWtuiqKhmathkqKujwcdaohSGjWaiCkmEDWqiOiWJs/cdTmT5oE9ySZrFVn + E1jH6KxVZwtWv8isVWdDv1UioNYRRDi9NVTGqF/l2K8EFx0GItJb7//Gm2gyvfW+AUhvnWpokuzeDtNb + nU2SeztIb3W2CO5tP711sgFKb/V1BBEcKA/TW+//BdJbpxqCJLkGdPkLyp4sd0l7ErQlXSFuoDwpzTV3 + jZB7ldJcIdPjNeZHAbwz7cimPCWfAadiM+CUcK6XYud6qTXzqVR8PlUvm/vVc3O/XoW/Jryyvya8Sn9N + eOV+TTjZzyD+h2UVOKIJ619NV9YHvafutH/93vV/vy1ueyhtnPzn8oQORj7h/9UWtdlcZKqpv/Zm739n + fbbYgNFzDt+y6rL8y1pKGycjZUPLR/45/5xuq2Z3SnN9RuYzt2LxxyuUdkp+uW7N1FlEp/WjQzMsh4i2 + lJ5s5LWnnXpK0rIvuqwvm1ql2W5XtH0GfAYXYwRO5gOAw/KL6aoCWrst0qLedR8tFlDJyF3+F/vVoPn4 + tcjtxUDogdhnt1mnivRYZMD9ESpd6s/2jPLCnhECdYQT5nnbN6eiNoniT/rOLOvFH3oSUo67q8qi7u01 + xmMrFqA4X1185Wsx7qz06Re9zJhmcc76VjZ1pUCi7XkC79KnR/uxtvk+WzfgUisPw/mVSl2K7iHXkURx + vp2uCTIbo+SopurKqEbJUS/1ilp0FdPsRF4/kzTKfVj9TJD6mTywfiZQ/UxW189kQf1MHlM/k6X1M3lc + /UyQ+pmI62cSqZ+JuH4mkfqZrKmfSaR+tqqXPj9HKcd9TP3kUZzvg+pnhMU5r6qfAYF3WVs/aQzn95j6 + yaM4X1H9vCs5qqh+3pUcVVo/p+IJu6k+0s13JBFhIhk5JkLOXOGTtrDZR9vLfl+Yd2b9emFegxYf8Dxp + 4ipZbamjV1vq7gsnXfMMgZpFaV2y/jMzn963w4/paa9PU+mzPCMWLIT2sqFFXfYmsbhpOfKPQkb9UbjE + sn7NqjIHW7JQ6VLhT/Mdkcdac8VmrlSwWZSNNU9yXe21lRoFYpe9IuKLkZN8fWeu9fARjs+P9OlT8jk9 + ZP2x6F5s/hZgQagpukmvkpFvSopa64ufdEUuRDtyiq+3JWYnId+RU3y1y/peXuiOnOR/76Toq3KkqqQU + /Rri6wii5NcQUjxhH7OnYOgWCX1hAQs8ktUmyZzL8pAYTj/ngATR8IQ5FyiiJoJwfEza1MprzyHmfaBS + YwjzLuDVYRnzTugV4iGOl1khYOU14hDzPmDpsYyJ00m/ehWLO4rX3R19XeiH9KWqAMZN4nKWr6ky7O2o + 26YF1HpvX42Ww01CctLiXYDSKpd2UUcEo3d39K/mV0UAYPefENp3m+mfLg43HhUuxazbZt4A2qy0WeMd + AgzELlt3pJV+L7gOyJQHBO1rCTIyQOCIKNYJ+VHRkxG8Xt8zJmYPJt6ELlMyXuXreOJtxGz5KANP8F16 + e0b6dTMH6l2gdKnHHr72V0nAGd5mQNIgcll2OcpjVtZwJXKVIXVIphRA78KQKa3wvjYkV9lHIeOOypBq + 7wQJ9C5kmMeiPBx7EXWQMlz4fleR+91u+2gLmKc1HgmsNmGd6e1dtUcgVwnFOeKcI8k5q4MApVUUre0E + 56dFDEt0bIOOIvYnnNafSFIlIFUeqUkvZd3/9BlC3UQeS/DQpJ+XA934VEWN/Q7CyF0+/tignhlvTS/u + H/lamgz2aSYygoc2HneRy3o/K/FZ+1qCjB7lXTSyXpNSNE/V1/HEr1LkV54JvNgQ0gn3Oc1Ml65c3Bsc + FS6l6hFC1Tvq7a6pFaC3+zuEXdtUCMHu7xK6yvxQkgPL7rqqgAa8SY+KgNLZmakgaBD5rByjuFc4L6o+ + M/8GIHeNQyredcfyAmAGgcPQ7+nqWKgePKCpzOGVeQtg9N6uut43iFzv7umP5dYkhNcf0GFMZA7PVNCL + yg7InXzXOKQ6O5tF32rVd5lZvBwA+lKXq9Iye0mrUiHtxkTl0XZFh4GMwGE0O9Waucj6DkGuwVQW8urG + /taN8q4yh6cbrHL3IbwWoZhin7O2LeuDAHxTOlQFVgsV1AsFP5tU8GxqdO9aMOXR15HEVZOp5jik47pp + VLMg0lMyIMXISf6qqUxzHNIRmcTkyUge0g/1ZCQPnLgUKn0qPqXQ15HEB9z/S2YSTvZ8xP2/aA7hZFf5 + /R+ZPTjZ4QH3/5J5fJM98fufmME32YDf/8TcPW/DsIZc2zXN/r4YKD67EoKSxyKqi/QMwtc2K1S62+5u + 3xEthvrCgNl3z8n96yT7Y6MC4QTBdwG/FXJEPktUAszZm/HPqw1URykxxb6Viog9EY/sd+GCZu/sembX + LYcCWWDPEVEs047YZgRd/DKCoHzap/bJDMG1CW4waqPk5xXkZ5L8bLbtMt1VFxT4VE3Rh9bJrEGFs0dt + nAwtNc8CFniYxdtW+xjIjJc6Z1WFLj0/TyJdl6817IgoVt9Aj/xAGDDhSb3v7JqG1y1qB64A7esI4m0V + 615we3jqCf3l0y/fnu33tHYexdBWKvtN+mKPCMN1uk5ltz2vfOhc6AOrttnyd/4ZjOeXlwczfGX7Mll1 + aDq97xmyIgm0y3X6L/KtNCP3+G1nlj+1k7HNGD+U184CPA/7oUFvf3/S+0B0V0pwjalpvft3mDtKXa4Z + FU/KtGyRx7enC4jDc1fbHYt3EDqVBlz72DLDskWtSmDonpGH/KbeD+OH56zX+8IGvj5w0GcFL/FOSANu + 1TQnlVblqUjzWtljAPEE4Z//+D8AMksHMdUEAA== EOF # PrivacyInfo.xcprivacy is not part of BoringSSL repo, inject it during pod installation diff --git a/templates/CMakeLists.txt.template b/templates/CMakeLists.txt.template index e94c05d06c1..a5ea02e73ef 100644 --- a/templates/CMakeLists.txt.template +++ b/templates/CMakeLists.txt.template @@ -371,6 +371,10 @@ if(UNIX) if(<%text>${CMAKE_SYSTEM_NAME} MATCHES "Linux") set(_gRPC_PLATFORM_LINUX ON) + if(NOT CMAKE_CROSSCOMPILING AND CMAKE_SIZEOF_VOID_P EQUAL 4) + message("+++ Enabling SSE2 for <%text>${CMAKE_SYSTEM_PROCESSOR}") + set(_gRPC_C_CXX_FLAGS "<%text>${_gRPC_C_CXX_FLAGS} -msse2") + endif() elseif(<%text>${CMAKE_SYSTEM_NAME} MATCHES "Darwin") set(_gRPC_PLATFORM_MAC ON) elseif(<%text>${CMAKE_SYSTEM_NAME} MATCHES "iOS") diff --git a/templates/gRPC-Core.podspec.template b/templates/gRPC-Core.podspec.template index e1ddf100bfd..c3a0b10b0f3 100644 --- a/templates/gRPC-Core.podspec.template +++ b/templates/gRPC-Core.podspec.template @@ -198,7 +198,7 @@ ss.libraries = 'z' ss.dependency "#{s.name}/Interface", version ss.dependency "#{s.name}/Privacy", version - ss.dependency 'BoringSSL-GRPC', '0.0.32' + ss.dependency 'BoringSSL-GRPC', '0.0.33' % for abseil_spec in grpc_abseil_specs: ss.dependency '${abseil_spec}', abseil_version % endfor diff --git a/templates/src/objective-c/BoringSSL-GRPC.podspec.template b/templates/src/objective-c/BoringSSL-GRPC.podspec.template index b770f9f6cc1..0d99cd189a8 100644 --- a/templates/src/objective-c/BoringSSL-GRPC.podspec.template +++ b/templates/src/objective-c/BoringSSL-GRPC.podspec.template @@ -70,7 +70,7 @@ Pod::Spec.new do |s| s.name = 'BoringSSL-GRPC' - version = '0.0.32' + version = '0.0.33' s.version = version s.summary = 'BoringSSL is a fork of OpenSSL that is designed to meet Google\'s needs.' # Adapted from the homepage: diff --git a/third_party/boringssl-with-bazel b/third_party/boringssl-with-bazel index ae72a4514c7..e14d29f68c2 160000 --- a/third_party/boringssl-with-bazel +++ b/third_party/boringssl-with-bazel @@ -1 +1 @@ -Subproject commit ae72a4514c7afd150596b0a80947f3ca9b8363b5 +Subproject commit e14d29f68c2d1b02e06f10c83b9b8ea4d061f8df diff --git a/tools/run_tests/sanity/check_submodules.sh b/tools/run_tests/sanity/check_submodules.sh index 0f45bfd0375..ec32590d1b9 100755 --- a/tools/run_tests/sanity/check_submodules.sh +++ b/tools/run_tests/sanity/check_submodules.sh @@ -28,7 +28,7 @@ cat <"$want_submodules" third_party/abseil-cpp 4a2c63365eff8823a5221db86ef490e828306f9d third_party/benchmark 344117638c8ff7e239044fd0fa7085839fc03021 third_party/bloaty 60209eb1ccc34d5deefb002d1b7f37545204f7f2 -third_party/boringssl-with-bazel ae72a4514c7afd150596b0a80947f3ca9b8363b5 +third_party/boringssl-with-bazel e14d29f68c2d1b02e06f10c83b9b8ea4d061f8df third_party/cares/cares 6360e96b5cf8e5980c887ce58ef727e53d77243a third_party/envoy-api 78f198cf96ecdc7120ef640406770aa01af775c4 third_party/googleapis 2f9af297c84c55c8b871ba4495e01ade42476c92 From ae131c991f4513d62c7df520219a848d9c100da6 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Mon, 11 Mar 2024 19:59:30 -0700 Subject: [PATCH 29/52] [client idleness] remove experiment (#35229) Closes #35229 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/35229 from markdroth:client_idleness_experiment_cleanup 65c570d07a2d74d550ea673bc037a2f71b34a431 PiperOrigin-RevId: 614882554 --- bazel/experiments.bzl | 1 - .../filters/channel_idle/channel_idle_filter.cc | 9 ++------- .../channel_idle/legacy_channel_idle_filter.cc | 9 ++------- src/core/lib/experiments/experiments.cc | 15 --------------- src/core/lib/experiments/experiments.h | 11 ----------- src/core/lib/experiments/experiments.yaml | 5 ----- src/core/lib/experiments/rollouts.yaml | 2 -- .../core/channel/minimal_stack_is_minimal_test.cc | 4 +--- 8 files changed, 5 insertions(+), 51 deletions(-) diff --git a/bazel/experiments.bzl b/bazel/experiments.bzl index 36ab8d6eb23..7e58d5cb29c 100644 --- a/bazel/experiments.bzl +++ b/bazel/experiments.bzl @@ -21,7 +21,6 @@ EXPERIMENT_ENABLES = { "call_status_override_on_cancellation": "call_status_override_on_cancellation", "call_v3": "call_v3", "canary_client_privacy": "canary_client_privacy", - "client_idleness": "client_idleness", "client_privacy": "client_privacy", "event_engine_client": "event_engine_client", "event_engine_dns": "event_engine_dns", diff --git a/src/core/ext/filters/channel_idle/channel_idle_filter.cc b/src/core/ext/filters/channel_idle/channel_idle_filter.cc index f2cbd0befb4..30155e5483c 100644 --- a/src/core/ext/filters/channel_idle/channel_idle_filter.cc +++ b/src/core/ext/filters/channel_idle/channel_idle_filter.cc @@ -64,12 +64,7 @@ const NoInterceptor ChannelIdleFilter::Call::OnServerToClientMessage; namespace { -// TODO(roth): This can go back to being a constant when the experiment -// is removed. -Duration DefaultIdleTimeout() { - if (IsClientIdlenessEnabled()) return Duration::Minutes(30); - return Duration::Infinity(); -} +constexpr Duration kDefaultIdleTimeout = Duration::Minutes(30); // If these settings change, make sure that we are not sending a GOAWAY for // inproc transport, since a GOAWAY to inproc ends up destroying the transport. @@ -92,7 +87,7 @@ namespace { Duration GetClientIdleTimeout(const ChannelArgs& args) { return args.GetDurationFromIntMillis(GRPC_ARG_CLIENT_IDLE_TIMEOUT_MS) - .value_or(DefaultIdleTimeout()); + .value_or(kDefaultIdleTimeout); } } // namespace diff --git a/src/core/ext/filters/channel_idle/legacy_channel_idle_filter.cc b/src/core/ext/filters/channel_idle/legacy_channel_idle_filter.cc index 8cb06f4b764..f9134a74df9 100644 --- a/src/core/ext/filters/channel_idle/legacy_channel_idle_filter.cc +++ b/src/core/ext/filters/channel_idle/legacy_channel_idle_filter.cc @@ -58,12 +58,7 @@ namespace grpc_core { namespace { -// TODO(roth): This can go back to being a constant when the experiment -// is removed. -Duration DefaultIdleTimeout() { - if (IsClientIdlenessEnabled()) return Duration::Minutes(30); - return Duration::Infinity(); -} +constexpr Duration kDefaultIdleTimeout = Duration::Minutes(30); // If these settings change, make sure that we are not sending a GOAWAY for // inproc transport, since a GOAWAY to inproc ends up destroying the transport. @@ -86,7 +81,7 @@ namespace { Duration GetClientIdleTimeout(const ChannelArgs& args) { return args.GetDurationFromIntMillis(GRPC_ARG_CLIENT_IDLE_TIMEOUT_MS) - .value_or(DefaultIdleTimeout()); + .value_or(kDefaultIdleTimeout); } } // namespace diff --git a/src/core/lib/experiments/experiments.cc b/src/core/lib/experiments/experiments.cc index 4ed2c0a792d..784735cd00c 100644 --- a/src/core/lib/experiments/experiments.cc +++ b/src/core/lib/experiments/experiments.cc @@ -36,9 +36,6 @@ const char* const additional_constraints_call_v3 = "{}"; const char* const description_canary_client_privacy = "If set, canary client privacy"; const char* const additional_constraints_canary_client_privacy = "{}"; -const char* const description_client_idleness = - "If enabled, client channel idleness is enabled by default."; -const char* const additional_constraints_client_idleness = "{}"; const char* const description_client_privacy = "If set, client privacy"; const char* const additional_constraints_client_privacy = "{}"; const char* const description_event_engine_client = @@ -194,8 +191,6 @@ const ExperimentMetadata g_experiment_metadata[] = { false, true}, {"canary_client_privacy", description_canary_client_privacy, additional_constraints_canary_client_privacy, nullptr, 0, false, false}, - {"client_idleness", description_client_idleness, - additional_constraints_client_idleness, nullptr, 0, true, true}, {"client_privacy", description_client_privacy, additional_constraints_client_privacy, nullptr, 0, false, false}, {"event_engine_client", description_event_engine_client, @@ -295,9 +290,6 @@ const char* const additional_constraints_call_v3 = "{}"; const char* const description_canary_client_privacy = "If set, canary client privacy"; const char* const additional_constraints_canary_client_privacy = "{}"; -const char* const description_client_idleness = - "If enabled, client channel idleness is enabled by default."; -const char* const additional_constraints_client_idleness = "{}"; const char* const description_client_privacy = "If set, client privacy"; const char* const additional_constraints_client_privacy = "{}"; const char* const description_event_engine_client = @@ -453,8 +445,6 @@ const ExperimentMetadata g_experiment_metadata[] = { false, true}, {"canary_client_privacy", description_canary_client_privacy, additional_constraints_canary_client_privacy, nullptr, 0, false, false}, - {"client_idleness", description_client_idleness, - additional_constraints_client_idleness, nullptr, 0, true, true}, {"client_privacy", description_client_privacy, additional_constraints_client_privacy, nullptr, 0, false, false}, {"event_engine_client", description_event_engine_client, @@ -554,9 +544,6 @@ const char* const additional_constraints_call_v3 = "{}"; const char* const description_canary_client_privacy = "If set, canary client privacy"; const char* const additional_constraints_canary_client_privacy = "{}"; -const char* const description_client_idleness = - "If enabled, client channel idleness is enabled by default."; -const char* const additional_constraints_client_idleness = "{}"; const char* const description_client_privacy = "If set, client privacy"; const char* const additional_constraints_client_privacy = "{}"; const char* const description_event_engine_client = @@ -712,8 +699,6 @@ const ExperimentMetadata g_experiment_metadata[] = { false, true}, {"canary_client_privacy", description_canary_client_privacy, additional_constraints_canary_client_privacy, nullptr, 0, false, false}, - {"client_idleness", description_client_idleness, - additional_constraints_client_idleness, nullptr, 0, true, true}, {"client_privacy", description_client_privacy, additional_constraints_client_privacy, nullptr, 0, false, false}, {"event_engine_client", description_event_engine_client, diff --git a/src/core/lib/experiments/experiments.h b/src/core/lib/experiments/experiments.h index a02aaab2157..1a51ba7799b 100644 --- a/src/core/lib/experiments/experiments.h +++ b/src/core/lib/experiments/experiments.h @@ -71,8 +71,6 @@ inline bool IsCallStatusOverrideOnCancellationEnabled() { } inline bool IsCallV3Enabled() { return false; } inline bool IsCanaryClientPrivacyEnabled() { return false; } -#define GRPC_EXPERIMENT_IS_INCLUDED_CLIENT_IDLENESS -inline bool IsClientIdlenessEnabled() { return true; } inline bool IsClientPrivacyEnabled() { return false; } inline bool IsEventEngineClientEnabled() { return false; } inline bool IsEventEngineDnsEnabled() { return false; } @@ -130,8 +128,6 @@ inline bool IsCallStatusOverrideOnCancellationEnabled() { } inline bool IsCallV3Enabled() { return false; } inline bool IsCanaryClientPrivacyEnabled() { return false; } -#define GRPC_EXPERIMENT_IS_INCLUDED_CLIENT_IDLENESS -inline bool IsClientIdlenessEnabled() { return true; } inline bool IsClientPrivacyEnabled() { return false; } inline bool IsEventEngineClientEnabled() { return false; } inline bool IsEventEngineDnsEnabled() { return false; } @@ -190,8 +186,6 @@ inline bool IsCallStatusOverrideOnCancellationEnabled() { } inline bool IsCallV3Enabled() { return false; } inline bool IsCanaryClientPrivacyEnabled() { return false; } -#define GRPC_EXPERIMENT_IS_INCLUDED_CLIENT_IDLENESS -inline bool IsClientIdlenessEnabled() { return true; } inline bool IsClientPrivacyEnabled() { return false; } inline bool IsEventEngineClientEnabled() { return false; } #define GRPC_EXPERIMENT_IS_INCLUDED_EVENT_ENGINE_DNS @@ -244,7 +238,6 @@ enum ExperimentIds { kExperimentIdCallStatusOverrideOnCancellation, kExperimentIdCallV3, kExperimentIdCanaryClientPrivacy, - kExperimentIdClientIdleness, kExperimentIdClientPrivacy, kExperimentIdEventEngineClient, kExperimentIdEventEngineDns, @@ -296,10 +289,6 @@ inline bool IsCallV3Enabled() { inline bool IsCanaryClientPrivacyEnabled() { return IsExperimentEnabled(kExperimentIdCanaryClientPrivacy); } -#define GRPC_EXPERIMENT_IS_INCLUDED_CLIENT_IDLENESS -inline bool IsClientIdlenessEnabled() { - return IsExperimentEnabled(kExperimentIdClientIdleness); -} #define GRPC_EXPERIMENT_IS_INCLUDED_CLIENT_PRIVACY inline bool IsClientPrivacyEnabled() { return IsExperimentEnabled(kExperimentIdClientPrivacy); diff --git a/src/core/lib/experiments/experiments.yaml b/src/core/lib/experiments/experiments.yaml index 95d63c0b46b..e67b4d4c92c 100644 --- a/src/core/lib/experiments/experiments.yaml +++ b/src/core/lib/experiments/experiments.yaml @@ -71,11 +71,6 @@ owner: ctiller@google.com requires: [promise_based_client_call, promise_based_server_call] test_tags: [core_end2end_test] -- name: client_idleness - description: If enabled, client channel idleness is enabled by default. - expiry: 2024/03/15 - owner: roth@google.com - test_tags: [] - name: client_privacy description: If set, client privacy diff --git a/src/core/lib/experiments/rollouts.yaml b/src/core/lib/experiments/rollouts.yaml index 94b54bbeadf..a971b0e4284 100644 --- a/src/core/lib/experiments/rollouts.yaml +++ b/src/core/lib/experiments/rollouts.yaml @@ -53,8 +53,6 @@ ios: broken posix: false windows: broken -- name: client_idleness - default: true - name: client_privacy default: false - name: event_engine_client diff --git a/test/core/channel/minimal_stack_is_minimal_test.cc b/test/core/channel/minimal_stack_is_minimal_test.cc index a501a02b12d..9212816e745 100644 --- a/test/core/channel/minimal_stack_is_minimal_test.cc +++ b/test/core/channel/minimal_stack_is_minimal_test.cc @@ -161,9 +161,7 @@ TEST(ChannelStackFilters, LooksAsExpected) { "http-server", "compression", "server_call_tracer", "connected"})); EXPECT_EQ(MakeStack(nullptr, no_args, GRPC_CLIENT_CHANNEL), - grpc_core::IsClientIdlenessEnabled() - ? std::vector({"client_idle", "client-channel"}) - : std::vector({"client-channel"})); + std::vector({"client_idle", "client-channel"})); } int main(int argc, char** argv) { From 750c451e96e671e5e69de0b4af7a252b8d25a0f0 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Tue, 12 Mar 2024 08:14:08 -0700 Subject: [PATCH 30/52] [fake stats plugin] add locking to avoid tsan failures (#36093) A couple of recent flakes: - https://btx-internal.corp.google.com/invocations/849676bf-0e4e-4ccf-9c17-21c19d6c0da5/targets/%2F%2Ftest%2Fcpp%2Fend2end:rls_end2end_test@poller%3Depoll1;config=b8fe4b9e3f3b31af34590e00748b2a7a095cedd3b431113a6063d7780bb24eca/log - https://btx-internal.corp.google.com/invocations/9515330c-a165-4ab5-81a1-3285d8a16bbb/targets/%2F%2Ftest%2Fcpp%2Fend2end%2Fxds:xds_wrr_end2end_test@experiment%3Dno_pick_first_happy_eyeballs;config=27047b7cfce2ec428f95a56e3bcafdc2cb72a602f6a43af1336b4c235e772213/log Closes #36093 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/36093 from markdroth:fake_stats_plugin_tsan_fix f504330684989bd0acbc7c0d0169b5e9fac5244d PiperOrigin-RevId: 615045684 --- test/core/util/fake_stats_plugin.h | 58 +++++++++++++++++++++++------- 1 file changed, 46 insertions(+), 12 deletions(-) diff --git a/test/core/util/fake_stats_plugin.h b/test/core/util/fake_stats_plugin.h index 04268ca694e..e6645f1df72 100644 --- a/test/core/util/fake_stats_plugin.h +++ b/test/core/util/fake_stats_plugin.h @@ -220,7 +220,8 @@ class FakeStatsPlugin : public StatsPlugin { return; } switch (descriptor.instrument_type) { - case GlobalInstrumentsRegistry::InstrumentType::kCounter: + case GlobalInstrumentsRegistry::InstrumentType::kCounter: { + MutexLock lock(&mu_); if (descriptor.value_type == GlobalInstrumentsRegistry::ValueType::kUInt64) { uint64_counters_.emplace(descriptor.index, descriptor); @@ -228,7 +229,9 @@ class FakeStatsPlugin : public StatsPlugin { double_counters_.emplace(descriptor.index, descriptor); } break; - case GlobalInstrumentsRegistry::InstrumentType::kHistogram: + } + case GlobalInstrumentsRegistry::InstrumentType::kHistogram: { + MutexLock lock(&mu_); if (descriptor.value_type == GlobalInstrumentsRegistry::ValueType::kUInt64) { uint64_histograms_.emplace(descriptor.index, descriptor); @@ -236,7 +239,9 @@ class FakeStatsPlugin : public StatsPlugin { double_histograms_.emplace(descriptor.index, descriptor); } break; - case GlobalInstrumentsRegistry::InstrumentType::kGauge: + } + case GlobalInstrumentsRegistry::InstrumentType::kGauge: { + MutexLock lock(&mu_); if (descriptor.value_type == GlobalInstrumentsRegistry::ValueType::kInt64) { int64_gauges_.emplace(descriptor.index, descriptor); @@ -244,7 +249,9 @@ class FakeStatsPlugin : public StatsPlugin { double_gauges_.emplace(descriptor.index, descriptor); } break; - case GlobalInstrumentsRegistry::InstrumentType::kCallbackGauge: + } + case GlobalInstrumentsRegistry::InstrumentType::kCallbackGauge: { + MutexLock lock(&callback_mu_); if (descriptor.value_type == GlobalInstrumentsRegistry::ValueType::kInt64) { int64_callback_gauges_.emplace(descriptor.index, descriptor); @@ -252,6 +259,7 @@ class FakeStatsPlugin : public StatsPlugin { double_callback_gauges_.emplace(descriptor.index, descriptor); } break; + } default: Crash("unknown instrument type"); } @@ -286,6 +294,7 @@ class FakeStatsPlugin : public StatsPlugin { this, handle.index, value, absl::StrJoin(label_values, ", ").c_str(), absl::StrJoin(optional_values, ", ").c_str()); + MutexLock lock(&mu_); auto iter = uint64_counters_.find(handle.index); if (iter == uint64_counters_.end()) return; iter->second.Add(value, label_values, optional_values); @@ -300,6 +309,7 @@ class FakeStatsPlugin : public StatsPlugin { this, handle.index, value, absl::StrJoin(label_values, ", ").c_str(), absl::StrJoin(optional_values, ", ").c_str()); + MutexLock lock(&mu_); auto iter = double_counters_.find(handle.index); if (iter == double_counters_.end()) return; iter->second.Add(value, label_values, optional_values); @@ -314,6 +324,7 @@ class FakeStatsPlugin : public StatsPlugin { this, handle.index, value, absl::StrJoin(label_values, ", ").c_str(), absl::StrJoin(optional_values, ", ").c_str()); + MutexLock lock(&mu_); auto iter = uint64_histograms_.find(handle.index); if (iter == uint64_histograms_.end()) return; iter->second.Record(value, label_values, optional_values); @@ -328,6 +339,7 @@ class FakeStatsPlugin : public StatsPlugin { this, handle.index, value, absl::StrJoin(label_values, ", ").c_str(), absl::StrJoin(optional_values, ", ").c_str()); + MutexLock lock(&mu_); auto iter = double_histograms_.find(handle.index); if (iter == double_histograms_.end()) return; iter->second.Record(value, label_values, optional_values); @@ -341,6 +353,7 @@ class FakeStatsPlugin : public StatsPlugin { this, handle.index, value, absl::StrJoin(label_values, ", ").c_str(), absl::StrJoin(optional_values, ", ").c_str()); + MutexLock lock(&mu_); auto iter = int64_gauges_.find(handle.index); if (iter == int64_gauges_.end()) return; iter->second.Set(value, label_values, optional_values); @@ -354,6 +367,7 @@ class FakeStatsPlugin : public StatsPlugin { this, handle.index, value, absl::StrJoin(label_values, ", ").c_str(), absl::StrJoin(optional_values, ", ").c_str()); + MutexLock lock(&mu_); auto iter = double_gauges_.find(handle.index); if (iter == double_gauges_.end()) return; iter->second.Set(value, label_values, optional_values); @@ -372,6 +386,7 @@ class FakeStatsPlugin : public StatsPlugin { GlobalInstrumentsRegistry::GlobalUInt64CounterHandle handle, absl::Span label_values, absl::Span optional_values) { + MutexLock lock(&mu_); auto iter = uint64_counters_.find(handle.index); if (iter == uint64_counters_.end()) { return absl::nullopt; @@ -382,6 +397,7 @@ class FakeStatsPlugin : public StatsPlugin { GlobalInstrumentsRegistry::GlobalDoubleCounterHandle handle, absl::Span label_values, absl::Span optional_values) { + MutexLock lock(&mu_); auto iter = double_counters_.find(handle.index); if (iter == double_counters_.end()) { return absl::nullopt; @@ -392,6 +408,7 @@ class FakeStatsPlugin : public StatsPlugin { GlobalInstrumentsRegistry::GlobalUInt64HistogramHandle handle, absl::Span label_values, absl::Span optional_values) { + MutexLock lock(&mu_); auto iter = uint64_histograms_.find(handle.index); if (iter == uint64_histograms_.end()) { return absl::nullopt; @@ -402,6 +419,7 @@ class FakeStatsPlugin : public StatsPlugin { GlobalInstrumentsRegistry::GlobalDoubleHistogramHandle handle, absl::Span label_values, absl::Span optional_values) { + MutexLock lock(&mu_); auto iter = double_histograms_.find(handle.index); if (iter == double_histograms_.end()) { return absl::nullopt; @@ -412,6 +430,7 @@ class FakeStatsPlugin : public StatsPlugin { GlobalInstrumentsRegistry::GlobalInt64GaugeHandle handle, absl::Span label_values, absl::Span optional_values) { + MutexLock lock(&mu_); auto iter = int64_gauges_.find(handle.index); if (iter == int64_gauges_.end()) { return absl::nullopt; @@ -422,6 +441,7 @@ class FakeStatsPlugin : public StatsPlugin { GlobalInstrumentsRegistry::GlobalDoubleGaugeHandle handle, absl::Span label_values, absl::Span optional_values) { + MutexLock lock(&mu_); auto iter = double_gauges_.find(handle.index); if (iter == double_gauges_.end()) { return absl::nullopt; @@ -440,6 +460,7 @@ class FakeStatsPlugin : public StatsPlugin { GlobalInstrumentsRegistry::GlobalCallbackInt64GaugeHandle handle, absl::Span label_values, absl::Span optional_values) { + MutexLock lock(&callback_mu_); auto iter = int64_callback_gauges_.find(handle.index); if (iter == int64_callback_gauges_.end()) { return absl::nullopt; @@ -450,6 +471,7 @@ class FakeStatsPlugin : public StatsPlugin { GlobalInstrumentsRegistry::GlobalCallbackDoubleGaugeHandle handle, absl::Span label_values, absl::Span optional_values) { + MutexLock lock(&callback_mu_); auto iter = double_callback_gauges_.find(handle.index); if (iter == double_callback_gauges_.end()) { return absl::nullopt; @@ -473,6 +495,7 @@ class FakeStatsPlugin : public StatsPlugin { this, handle.index, value, absl::StrJoin(label_values, ", ").c_str(), absl::StrJoin(optional_values, ", ").c_str()); + MutexLock lock(&plugin_.callback_mu_); auto iter = plugin_.int64_callback_gauges_.find(handle.index); if (iter == plugin_.int64_callback_gauges_.end()) return; iter->second.Set(value, label_values, optional_values); @@ -489,6 +512,7 @@ class FakeStatsPlugin : public StatsPlugin { this, handle.index, value, absl::StrJoin(label_values, ", ").c_str(), absl::StrJoin(optional_values, ", ").c_str()); + MutexLock lock(&plugin_.callback_mu_); auto iter = plugin_.double_callback_gauges_.find(handle.index); if (iter == plugin_.double_callback_gauges_.end()) return; iter->second.Set(value, label_values, optional_values); @@ -621,14 +645,24 @@ class FakeStatsPlugin : public StatsPlugin { absl::AnyInvocable channel_filter_; // Instruments. - absl::flat_hash_map> uint64_counters_; - absl::flat_hash_map> double_counters_; - absl::flat_hash_map> uint64_histograms_; - absl::flat_hash_map> double_histograms_; - absl::flat_hash_map> int64_gauges_; - absl::flat_hash_map> double_gauges_; - absl::flat_hash_map> int64_callback_gauges_; - absl::flat_hash_map> double_callback_gauges_; + Mutex mu_; + absl::flat_hash_map> uint64_counters_ + ABSL_GUARDED_BY(&mu_); + absl::flat_hash_map> double_counters_ + ABSL_GUARDED_BY(&mu_); + absl::flat_hash_map> uint64_histograms_ + ABSL_GUARDED_BY(&mu_); + absl::flat_hash_map> double_histograms_ + ABSL_GUARDED_BY(&mu_); + absl::flat_hash_map> int64_gauges_ + ABSL_GUARDED_BY(&mu_); + absl::flat_hash_map> double_gauges_ + ABSL_GUARDED_BY(&mu_); + Mutex callback_mu_; + absl::flat_hash_map> int64_callback_gauges_ + ABSL_GUARDED_BY(&callback_mu_); + absl::flat_hash_map> double_callback_gauges_ + ABSL_GUARDED_BY(&callback_mu_); std::set callbacks_; }; From e34c20cd13da079f9ce889b4f5f8c1607d69009c Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Tue, 12 Mar 2024 10:21:34 -0700 Subject: [PATCH 31/52] [CI] Refactor android_ndk docker (#36091) Refactored android-ndk docker image to have - Replaced a full Android SDK with a Android command line tool (generally good for docker usage) and bumped its version to the latest among ones that support JDK 11 - Upgraded Debian from 10 to 11 - Upgraded Java SDK 8 to 11 - Upgraded Google CLI to the latest Closes #36091 PiperOrigin-RevId: 615088973 --- .../test/android_ndk/Dockerfile.template | 40 +++++++------------ .../dockerimage_current_versions.bzl | 2 +- .../test/android_ndk.current_version | 2 +- tools/dockerfile/test/android_ndk/Dockerfile | 40 +++++++------------ 4 files changed, 32 insertions(+), 52 deletions(-) diff --git a/templates/tools/dockerfile/test/android_ndk/Dockerfile.template b/templates/tools/dockerfile/test/android_ndk/Dockerfile.template index 4b3819ad06e..2b40d504c29 100644 --- a/templates/tools/dockerfile/test/android_ndk/Dockerfile.template +++ b/templates/tools/dockerfile/test/android_ndk/Dockerfile.template @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - FROM debian:10 + FROM debian:11 <%include file="../../apt_get_basic.include"/> <%include file="../../run_tests_python_deps.include"/> @@ -22,36 +22,26 @@ <%include file="../../run_tests_addons.include"/> RUN apt update && apt upgrade -y - # Java required by Android SDK (using Eclipse Temurin Package) - RUN apt install -y wget apt-transport-https && ${'\\'} - mkdir -p /etc/apt/keyrings && ${'\\'} - wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | tee /etc/apt/keyrings/adoptium.asc && ${'\\'} - echo "deb [signed-by=/etc/apt/keyrings/adoptium.asc] https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | tee /etc/apt/sources.list.d/adoptium.list - RUN apt update && apt install -y temurin-8-jdk - - RUN apt-get -y autoremove && ${'\\'} - rm -rf /var/lib/apt/lists/* + # Java + RUN apt-get install -y openjdk-11-jdk # Install Android SDK - ENV ANDROID_SDK_VERSION 4333796 - RUN mkdir -p /opt/android-sdk && cd /opt/android-sdk && ${'\\'} - wget -q https://dl.google.com/android/repository/sdk-tools-linux-$ANDROID_SDK_VERSION.zip && ${'\\'} - unzip -q sdk-tools-linux-$ANDROID_SDK_VERSION.zip && ${'\\'} - rm sdk-tools-linux-$ANDROID_SDK_VERSION.zip - ENV ANDROID_SDK_PATH /opt/android-sdk ENV ANDROID_HOME /opt/android-sdk - RUN yes | $ANDROID_SDK_PATH/tools/bin/sdkmanager --licenses # accept all licenses - - # Install Android NDK & CMake - # This is not required but desirable to reduce the time to download and the chance of download failure. - RUN mkdir -p ~/.android && touch ~/.android/repositories.cfg - RUN $ANDROID_SDK_PATH/tools/bin/sdkmanager 'ndk;25.1.8937393' 'cmake;3.22.1' + RUN mkdir -p $ANDROID_HOME && ${'\\'} + wget -O cmd.zip https://dl.google.com/android/repository/commandlinetools-linux-9477386_latest.zip && ${'\\'} + unzip cmd.zip && rm cmd.zip && ${'\\'} + yes | ./cmdline-tools/bin/sdkmanager --sdk_root=$ANDROID_HOME --licenses && ${'\\'} + ./cmdline-tools/bin/sdkmanager --sdk_root=$ANDROID_HOME 'tools' && ${'\\'} + ./cmdline-tools/bin/sdkmanager --sdk_root=$ANDROID_HOME 'build-tools;30.0.2' && ${'\\'} + ./cmdline-tools/bin/sdkmanager --sdk_root=$ANDROID_HOME 'platforms;android-26' && ${'\\'} + ./cmdline-tools/bin/sdkmanager --sdk_root=$ANDROID_HOME 'ndk;25.1.8937393' && ${'\\'} + ./cmdline-tools/bin/sdkmanager --sdk_root=$ANDROID_HOME 'cmake;3.22.1' # Install gcloud - RUN curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-407.0.0-linux-x86_64.tar.gz && ${'\\'} - tar -xf google-cloud-cli-407.0.0-linux-x86_64.tar.gz && ${'\\'} + RUN wget -O google-cloud-cli.tar.gz https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-467.0.0-linux-x86_64.tar.gz && ${'\\'} + tar -xf google-cloud-cli.tar.gz && ${'\\'} ./google-cloud-sdk/install.sh --bash-completion=false --path-update=true && ${'\\'} - rm -rf google-cloud-cli-407.0.0-linux-x86_64.tar.gz + rm -rf google-cloud-cli.tar.gz RUN ln -s /google-cloud-sdk/bin/gcloud /usr/local/bin/gcloud # Define the default command. diff --git a/tools/bazelify_tests/dockerimage_current_versions.bzl b/tools/bazelify_tests/dockerimage_current_versions.bzl index c30cc5ae966..70dcfc40ae5 100644 --- a/tools/bazelify_tests/dockerimage_current_versions.bzl +++ b/tools/bazelify_tests/dockerimage_current_versions.bzl @@ -87,7 +87,7 @@ DOCKERIMAGE_CURRENT_VERSIONS = { "tools/dockerfile/interoptest/grpc_interop_pythonasyncio.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_pythonasyncio@sha256:47127a7863097b436613885a8866a2ef055470452838ceebb31f692ac88ac1d1", "tools/dockerfile/interoptest/grpc_interop_ruby.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_ruby@sha256:efd7f41a736dd4b8f73b32f5215b86f6bfe9013c422dfcd77978de0253aaee45", "tools/dockerfile/interoptest/lb_interop_fake_servers.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/lb_interop_fake_servers@sha256:b89a51dd9147e1293f50ee64dd719fce5929ca7894d3770a3d80dbdecb99fd52", - "tools/dockerfile/test/android_ndk.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/android_ndk@sha256:2866e815ceaf7407a9017842b86c1bf1efc96abe77ecff88e932aa1b1ddf727e", + "tools/dockerfile/test/android_ndk.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/android_ndk@sha256:ab154ecb062af2111d2d3550c4d3da3384201d9893bbd37d49e8160fc34bc137", "tools/dockerfile/test/bazel.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/bazel@sha256:32bde2dcb2087f2a32afab59e4dfedf7e8c76a52c69881f63a239d311f0e5ecf", "tools/dockerfile/test/bazel_arm64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/bazel_arm64@sha256:3b087387c44dee405c1b80d6ff50994e6d8e90a4ef67cc94b4291f1a29c0ef41", "tools/dockerfile/test/binder_transport_apk.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/binder_transport_apk@sha256:bf60a187cd2ce1abe8b4f32ae6479040a72ca6aa789cd5ab509f60ceb37a41f9", diff --git a/tools/dockerfile/test/android_ndk.current_version b/tools/dockerfile/test/android_ndk.current_version index ee0ccad695b..94dceb602b7 100644 --- a/tools/dockerfile/test/android_ndk.current_version +++ b/tools/dockerfile/test/android_ndk.current_version @@ -1 +1 @@ -us-docker.pkg.dev/grpc-testing/testing-images-public/android_ndk:7642d2e940a8b14b0f3b63798642869e544d8e01@sha256:2866e815ceaf7407a9017842b86c1bf1efc96abe77ecff88e932aa1b1ddf727e \ No newline at end of file +us-docker.pkg.dev/grpc-testing/testing-images-public/android_ndk:5b30710d3f4e0ad76921b0a9216154ac26f67460@sha256:ab154ecb062af2111d2d3550c4d3da3384201d9893bbd37d49e8160fc34bc137 \ No newline at end of file diff --git a/tools/dockerfile/test/android_ndk/Dockerfile b/tools/dockerfile/test/android_ndk/Dockerfile index 19973c1a18f..514d81bdf6f 100644 --- a/tools/dockerfile/test/android_ndk/Dockerfile +++ b/tools/dockerfile/test/android_ndk/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM debian:10 +FROM debian:11 #================= # Basic C core dependencies @@ -98,36 +98,26 @@ RUN mkdir /var/local/jenkins RUN apt update && apt upgrade -y -# Java required by Android SDK (using Eclipse Temurin Package) -RUN apt install -y wget apt-transport-https && \ - mkdir -p /etc/apt/keyrings && \ - wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | tee /etc/apt/keyrings/adoptium.asc && \ - echo "deb [signed-by=/etc/apt/keyrings/adoptium.asc] https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | tee /etc/apt/sources.list.d/adoptium.list -RUN apt update && apt install -y temurin-8-jdk - -RUN apt-get -y autoremove && \ - rm -rf /var/lib/apt/lists/* +# Java +RUN apt-get install -y openjdk-11-jdk # Install Android SDK -ENV ANDROID_SDK_VERSION 4333796 -RUN mkdir -p /opt/android-sdk && cd /opt/android-sdk && \ - wget -q https://dl.google.com/android/repository/sdk-tools-linux-$ANDROID_SDK_VERSION.zip && \ - unzip -q sdk-tools-linux-$ANDROID_SDK_VERSION.zip && \ - rm sdk-tools-linux-$ANDROID_SDK_VERSION.zip -ENV ANDROID_SDK_PATH /opt/android-sdk ENV ANDROID_HOME /opt/android-sdk -RUN yes | $ANDROID_SDK_PATH/tools/bin/sdkmanager --licenses # accept all licenses - -# Install Android NDK & CMake -# This is not required but desirable to reduce the time to download and the chance of download failure. -RUN mkdir -p ~/.android && touch ~/.android/repositories.cfg -RUN $ANDROID_SDK_PATH/tools/bin/sdkmanager 'ndk;25.1.8937393' 'cmake;3.22.1' +RUN mkdir -p $ANDROID_HOME && \ + wget -O cmd.zip https://dl.google.com/android/repository/commandlinetools-linux-9477386_latest.zip && \ + unzip cmd.zip && rm cmd.zip && \ + yes | ./cmdline-tools/bin/sdkmanager --sdk_root=$ANDROID_HOME --licenses && \ + ./cmdline-tools/bin/sdkmanager --sdk_root=$ANDROID_HOME 'tools' && \ + ./cmdline-tools/bin/sdkmanager --sdk_root=$ANDROID_HOME 'build-tools;30.0.2' && \ + ./cmdline-tools/bin/sdkmanager --sdk_root=$ANDROID_HOME 'platforms;android-26' && \ + ./cmdline-tools/bin/sdkmanager --sdk_root=$ANDROID_HOME 'ndk;25.1.8937393' && \ + ./cmdline-tools/bin/sdkmanager --sdk_root=$ANDROID_HOME 'cmake;3.22.1' # Install gcloud -RUN curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-407.0.0-linux-x86_64.tar.gz && \ - tar -xf google-cloud-cli-407.0.0-linux-x86_64.tar.gz && \ +RUN wget -O google-cloud-cli.tar.gz https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-467.0.0-linux-x86_64.tar.gz && \ + tar -xf google-cloud-cli.tar.gz && \ ./google-cloud-sdk/install.sh --bash-completion=false --path-update=true && \ - rm -rf google-cloud-cli-407.0.0-linux-x86_64.tar.gz + rm -rf google-cloud-cli.tar.gz RUN ln -s /google-cloud-sdk/bin/gcloud /usr/local/bin/gcloud # Define the default command. From b7f92176339860ab3817742fafc1ac8cece467cf Mon Sep 17 00:00:00 2001 From: Gregory Cooke Date: Tue, 12 Mar 2024 12:47:56 -0700 Subject: [PATCH 32/52] [Security - Revocation] Refactor how CRLs are checked internally (#36031) This PR changes how CRLs are handled purely internally. After discussing with davidben@, there are various problems with the `X509_STORE_set_get_crl` API and we shouldn't use it. This change keeps the behavior and external API the same, but instead of bulk pushing CRL information into OpenSSL, we instead iterate through the built chain and check each certificate for revocation, as well as doing the CRL validation ourselves. Closes #36031 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/36031 from gtcooke94:CrlInternalRefactor 5f4c81664835cc4382010dd0dc5e9218a7b3a606 PiperOrigin-RevId: 615139682 --- src/core/tsi/ssl_transport_security.cc | 236 +++++++++++++----- test/core/tsi/BUILD | 3 + .../tsi/crl_ssl_transport_security_test.cc | 50 ++++ test/core/tsi/test_creds/crl_data/BUILD | 1 + test/core/tsi/test_creds/crl_data/README | 6 + test/core/tsi/test_creds/crl_data/crls/BUILD | 1 + .../tsi/test_creds/crl_data/crls/evil.crl | 12 + test/core/tsi/test_creds/crl_data/evil_ca.cnf | 43 ++++ test/core/tsi/test_creds/crl_data/evil_ca.key | 52 ++-- test/core/tsi/test_creds/crl_data/evil_ca.pem | 30 +-- .../tsi/test_creds/crl_data/evil_ca_gen.sh | 18 +- 11 files changed, 349 insertions(+), 103 deletions(-) create mode 100644 test/core/tsi/test_creds/crl_data/crls/evil.crl create mode 100644 test/core/tsi/test_creds/crl_data/evil_ca.cnf diff --git a/src/core/tsi/ssl_transport_security.cc b/src/core/tsi/ssl_transport_security.cc index 661e27c5fe8..1ee9faf284a 100644 --- a/src/core/tsi/ssl_transport_security.cc +++ b/src/core/tsi/ssl_transport_security.cc @@ -72,6 +72,7 @@ #define TSI_SSL_MAX_PROTECTED_FRAME_SIZE_UPPER_BOUND 16384 #define TSI_SSL_MAX_PROTECTED_FRAME_SIZE_LOWER_BOUND 1024 #define TSI_SSL_HANDSHAKER_OUTGOING_BUFFER_INITIAL_SIZE 1024 +const size_t kMaxChainLength = 100; // Putting a macro like this and littering the source file with #if is really // bad practice. @@ -911,13 +912,7 @@ static int NullVerifyCallback(X509_STORE_CTX* /*ctx*/, void* /*arg*/) { } static int RootCertExtractCallback(X509_STORE_CTX* ctx, void* /*arg*/) { - int ret = X509_verify_cert(ctx); - if (ret <= 0) { - // Verification failed. We shouldn't expect to have a verified chain, so - // there is no need to attempt to extract the root cert from it. - return ret; - } - + int ret = 1; // Verification was successful. Get the verified chain from the X509_STORE_CTX // and put the root on the SSL object so that we have access to it when // populating the tsi_peer. On error extracting the root, we return success @@ -977,37 +972,42 @@ static int RootCertExtractCallback(X509_STORE_CTX* ctx, void* /*arg*/) { return ret; } -// X509_STORE_set_get_crl() sets the function to get the crl for a given -// certificate x. When found, the crl must be assigned to *crl. This function -// must return 0 on failure and 1 on success. If no function to get the issuer -// is provided, the internal default function will be used instead. -static int GetCrlFromProvider(X509_STORE_CTX* ctx, X509_CRL** crl_out, - X509* cert) { +static grpc_core::experimental::CrlProvider* GetCrlProvider( + X509_STORE_CTX* ctx) { ERR_clear_error(); int ssl_index = SSL_get_ex_data_X509_STORE_CTX_idx(); if (ssl_index < 0) { char err_str[256]; ERR_error_string_n(ERR_get_error(), err_str, sizeof(err_str)); - gpr_log(GPR_ERROR, + gpr_log(GPR_INFO, "error getting the SSL index from the X509_STORE_CTX while looking " "up Crl: %s", err_str); - return 0; + return nullptr; } SSL* ssl = static_cast(X509_STORE_CTX_get_ex_data(ctx, ssl_index)); if (ssl == nullptr) { - gpr_log(GPR_ERROR, + gpr_log(GPR_INFO, "error while fetching from CrlProvider. SSL object is null"); - return 0; + return nullptr; } SSL_CTX* ssl_ctx = SSL_get_SSL_CTX(ssl); auto* provider = static_cast( SSL_CTX_get_ex_data(ssl_ctx, g_ssl_ctx_ex_crl_provider_index)); + return provider; +} +// If a CRL is returned, the caller is the owner of the CRL and must make sure +// it is freed. +static absl::StatusOr GetCrlFromProvider( + grpc_core::experimental::CrlProvider* provider, X509* cert) { + if (provider == nullptr) { + return absl::InvalidArgumentError("CrlProvider is null."); + } absl::StatusOr issuer_name = grpc_core::IssuerFromCert(cert); if (!issuer_name.ok()) { gpr_log(GPR_INFO, "Could not get certificate issuer name"); - return 0; + return absl::InvalidArgumentError(issuer_name.status().message()); } absl::StatusOr akid = grpc_core::AkidFromCertificate(cert); std::string akid_to_use; @@ -1026,28 +1026,142 @@ static int GetCrlFromProvider(X509_STORE_CTX* ctx, X509_CRL** crl_out, // and behave how we want for a missing CRL. // It is important to treat missing CRLs and empty CRLs differently. if (internal_crl == nullptr) { - return 0; + return absl::NotFoundError("Could not find Crl related to certificate."); } X509_CRL* crl = std::static_pointer_cast(internal_crl) ->crl(); - X509_CRL* copy = X509_CRL_dup(crl); - *crl_out = copy; + return X509_CRL_dup(crl); +} + +// Perform the validation checks in RFC5280 6.3.3 to ensure the given CRL is +// valid +// returns true if the Crl is valid, false otherwise +static bool ValidateCrl(X509* cert, X509* issuer, X509_CRL* crl) { + bool valid = true; + // RFC5280 6.3.3 + // 6.3.3a we do not support distribution points + // 6.3.3b verify issuer and scope + valid = grpc_core::VerifyCrlCertIssuerNamesMatch(crl, cert); + if (!valid) { + return valid; + } + valid = grpc_core::HasCrlSignBit(issuer); + if (!valid) { + return valid; + } + // 6.3.3c Not supporting deltas + // 6.3.3d Not supporting reasons masks + // 6.3.3e Not supporting reasons masks + // 6.3.3f We only support direct CRLs so these paths are by definition the + // same. + // 6.3.3g Verify CRL Signature + valid = grpc_core::VerifyCrlSignature(crl, issuer); + return valid; +} + +// Check if a given certificate is revoked +// Returns 1 if the certificate is not revoked, 0 if the certificate is revoked +static int CheckCertRevocation(grpc_core::experimental::CrlProvider* provider, + X509* cert, X509* issuer) { + auto crl = GetCrlFromProvider(provider, cert); + // Not finding a CRL is a specific behavior. Per RFC5280, not having a CRL to + // check for a given certificate means that we cannot know for certain if the + // status is Revoked or Unrevoked and instead is Undetermined. How a user + // handles an Undetermined CRL is up to them. We use absl::IsNotFound as an + // analogue for not finding the Crl from the provider, thus the certificate in + // question is Undetermined. + if (absl::IsNotFound(crl.status())) { + // TODO(gtcooke94) knob for undetermined being revoked or unrevoked. By + // default, unrevoked. + return 1; + } else if (!crl.ok()) { + // This is an unexpected error, return false + return 0; + } + // Validate the crl + // RFC5280 6.3.3(a-i) + if (!ValidateCrl(cert, issuer, *crl)) { + X509_CRL_free(*crl); + return 0; + } + + // RFC5280 6.3.3j Actually check revocation + // Look for serial number of certificate in CRL X509_REVOKED* rev = + // nullptr; + X509_REVOKED* rev; + if (X509_CRL_get0_by_cert(*crl, &rev, cert)) { + // cert is revoked + X509_CRL_free(*crl); + return 0; + } + // The certificate is not revoked + // RFC5280k - Not supported + // RFC5280l - Not supported + X509_CRL_free(*crl); return 1; } -// When using CRL Providers, this function used to override the default -// `check_crl` function in OpenSSL using `X509_STORE_set_check_crl`. -// CrlProviders put the onus on the users to provide the CRLs that they want to -// provide, and because we override default CRL fetching behavior, we can expect -// some of these verification checks to fails for custom CRL providers as well. -// Thus, we need a passthrough to indicate to OpenSSL that we've provided a CRL -// and we are good with it. -static int CheckCrlPassthrough(X509_STORE_CTX* /*ctx*/, X509_CRL* /*crl*/) { +// Checks each certificate in the chain for revocation +// returns 0 if any cert in the chain is revoked, 1 otherwise. +static int CheckChainRevocation( + X509_STORE_CTX* ctx, grpc_core::experimental::CrlProvider* provider) { +#if OPENSSL_VERSION_NUMBER >= 0x10100000 + STACK_OF(X509)* chain = X509_STORE_CTX_get0_chain(ctx); +#else + STACK_OF(X509)* chain = X509_STORE_CTX_get_chain(ctx); +#endif + if (chain == nullptr) { + return 0; + } + // BoringSSL returns a size_t (unsigned), while OpenSSL returns an int + // (signed). In OpenSSL, a -1 can indicate a problem. By forcing it into a + // size_t, a -1 return will result in the chain_length being a very large + // number, so it will still fail this check because that very large number + // will be >= kMaxChainLength + size_t chain_length = sk_X509_num(chain); + if (chain_length > kMaxChainLength || chain_length == 0) { + return 0; + } + // Loop to < chain_length - 1 because the last cert is the trust anchor/root + // which cannot be revoked + for (size_t i = 0; i < chain_length - 1; i++) { + X509* cert = sk_X509_value(chain, i); + X509* issuer = sk_X509_value(chain, i + 1); + int ret = CheckCertRevocation(provider, cert, issuer); + if (ret != 1) { + return ret; + } + } return 1; } +// The custom verification function to set in OpenSSL using +// X509_set_cert_verify_callback. This calls the standard OpenSSL procedure +// (X509_verify_cert), then also extracts the root certificate in the built +// chain and does revocation checks when a user has configured CrlProviders. +// returns 1 on success, indicating a trusted chain to a root of trust was +// found, 0 if a trusted chain could not be built. +static int CustomVerificationFunction(X509_STORE_CTX* ctx, void* arg) { + int ret = X509_verify_cert(ctx); + if (ret <= 0) { + // Verification failed. We shouldn't expect to have a verified chain, so + // there is no need to attempt to extract the root cert from it, check for + // revocation, or check anything else. + return ret; + } + grpc_core::experimental::CrlProvider* provider = GetCrlProvider(ctx); + if (provider != nullptr) { + ret = CheckChainRevocation(ctx, provider); + } + if (ret <= 0) { + // Something has failed return the failure + return ret; + } + return RootCertExtractCallback(ctx, arg); +} + // Sets the min and max TLS version of |ssl_context| to |min_tls_version| and // |max_tls_version|, respectively. Calling this method is a no-op when using // OpenSSL versions < 1.1. @@ -1069,9 +1183,9 @@ static tsi_result tsi_set_min_and_max_tls_versions( SSL_CTX_set_min_proto_version(ssl_context, TLS1_2_VERSION); break; #if defined(TLS1_3_VERSION) - // If the library does not support TLS 1.3 and the caller requests a minimum - // of TLS 1.3, then return an error because the caller's request cannot be - // satisfied. + // If the library does not support TLS 1.3 and the caller requests a + // minimum of TLS 1.3, then return an error because the caller's request + // cannot be satisfied. case tsi_tls_version::TSI_TLS1_3: SSL_CTX_set_min_proto_version(ssl_context, TLS1_3_VERSION); break; @@ -1131,6 +1245,12 @@ tsi_ssl_root_certs_store* tsi_ssl_root_certs_store_create( gpr_free(root_store); return nullptr; } +#if OPENSSL_VERSION_NUMBER >= 0x10100000 + X509_VERIFY_PARAM* param = X509_STORE_get0_param(root_store->store); +#else + X509_VERIFY_PARAM* param = root_store->store->param; +#endif + X509_VERIFY_PARAM_set_depth(param, kMaxChainLength); return root_store; } @@ -1586,8 +1706,8 @@ static tsi_result ssl_bytes_remaining(tsi_ssl_handshaker* impl, *bytes_remaining = static_cast(gpr_malloc(bytes_in_ssl)); int bytes_read = BIO_read(SSL_get_rbio(impl->ssl), *bytes_remaining, static_cast(bytes_in_ssl)); - // If an unexpected number of bytes were read, return an error status and free - // all of the bytes that were read. + // If an unexpected number of bytes were read, return an error status and + // free all of the bytes that were read. if (bytes_read < 0 || static_cast(bytes_read) != bytes_in_ssl) { gpr_log(GPR_ERROR, "Failed to read the expected number of bytes from SSL object."); @@ -1662,16 +1782,16 @@ static tsi_result ssl_handshaker_next(tsi_handshaker* self, impl, remaining_bytes_to_write_to_openssl, &bytes_written_to_openssl, error); // As long as the BIO is full, drive the SSL handshake to consume bytes - // from the BIO. If the SSL handshake returns any bytes, write them to the - // peer. + // from the BIO. If the SSL handshake returns any bytes, write them to + // the peer. while (status == TSI_DRAIN_BUFFER) { status = ssl_handshaker_write_output_buffer(self, &bytes_written, error); if (status != TSI_OK) return status; status = ssl_handshaker_do_handshake(impl, error); } - // Move the pointer to the first byte not yet successfully written to the - // BIO. + // Move the pointer to the first byte not yet successfully written to + // the BIO. remaining_bytes_to_write_to_openssl_size -= bytes_written_to_openssl; remaining_bytes_to_write_to_openssl += bytes_written_to_openssl; } @@ -1687,9 +1807,9 @@ static tsi_result ssl_handshaker_next(tsi_handshaker* self, *handshaker_result = nullptr; } else { // Any bytes that remain in |impl->ssl|'s read BIO after the handshake is - // complete must be extracted and set to the unused bytes of the handshaker - // result. This indicates to the gRPC stack that there are bytes from the - // peer that must be processed. + // complete must be extracted and set to the unused bytes of the + // handshaker result. This indicates to the gRPC stack that there are + // bytes from the peer that must be processed. unsigned char* unused_bytes = nullptr; size_t unused_bytes_size = 0; status = @@ -1704,8 +1824,8 @@ static tsi_result ssl_handshaker_next(tsi_handshaker* self, status = ssl_handshaker_result_create(impl, unused_bytes, unused_bytes_size, handshaker_result, error); if (status == TSI_OK) { - // Indicates that the handshake has completed and that a handshaker_result - // has been created. + // Indicates that the handshake has completed and that a + // handshaker_result has been created. self->handshaker_result_created = true; } } @@ -2152,6 +2272,15 @@ tsi_result tsi_create_ssl_client_handshaker_factory_with_options( result = ssl_ctx_load_verification_certs( ssl_context, options->pem_root_certs, strlen(options->pem_root_certs), nullptr); + X509_STORE* cert_store = SSL_CTX_get_cert_store(ssl_context); +#if OPENSSL_VERSION_NUMBER >= 0x10100000 + X509_VERIFY_PARAM* param = X509_STORE_get0_param(cert_store); + +#else + X509_VERIFY_PARAM* param = cert_store->param; +#endif + + X509_VERIFY_PARAM_set_depth(param, kMaxChainLength); if (result != TSI_OK) { gpr_log(GPR_ERROR, "Cannot load server root certificates."); break; @@ -2189,21 +2318,13 @@ tsi_result tsi_create_ssl_client_handshaker_factory_with_options( if (options->skip_server_certificate_verification) { SSL_CTX_set_cert_verify_callback(ssl_context, NullVerifyCallback, nullptr); } else { - SSL_CTX_set_cert_verify_callback(ssl_context, RootCertExtractCallback, + SSL_CTX_set_cert_verify_callback(ssl_context, CustomVerificationFunction, nullptr); } - #if OPENSSL_VERSION_NUMBER >= 0x10100000 && !defined(LIBRESSL_VERSION_NUMBER) if (options->crl_provider != nullptr) { SSL_CTX_set_ex_data(impl->ssl_context, g_ssl_ctx_ex_crl_provider_index, options->crl_provider.get()); - X509_STORE* cert_store = SSL_CTX_get_cert_store(impl->ssl_context); - X509_STORE_set_get_crl(cert_store, GetCrlFromProvider); - X509_STORE_set_check_crl(cert_store, CheckCrlPassthrough); - X509_STORE_set_verify_cb(cert_store, verify_cb); - X509_VERIFY_PARAM* param = X509_STORE_get0_param(cert_store); - X509_VERIFY_PARAM_set_flags( - param, X509_V_FLAG_CRL_CHECK | X509_V_FLAG_CRL_CHECK_ALL); } else if (options->crl_directory != nullptr && strcmp(options->crl_directory, "") != 0) { X509_STORE* cert_store = SSL_CTX_get_cert_store(ssl_context); @@ -2379,7 +2500,7 @@ tsi_result tsi_create_ssl_server_handshaker_factory_with_options( case TSI_REQUEST_CLIENT_CERTIFICATE_AND_VERIFY: SSL_CTX_set_verify(impl->ssl_contexts[i], SSL_VERIFY_PEER, nullptr); SSL_CTX_set_cert_verify_callback(impl->ssl_contexts[i], - RootCertExtractCallback, nullptr); + CustomVerificationFunction, nullptr); break; case TSI_REQUEST_AND_REQUIRE_CLIENT_CERTIFICATE_BUT_DONT_VERIFY: SSL_CTX_set_verify(impl->ssl_contexts[i], @@ -2393,7 +2514,7 @@ tsi_result tsi_create_ssl_server_handshaker_factory_with_options( SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, nullptr); SSL_CTX_set_cert_verify_callback(impl->ssl_contexts[i], - RootCertExtractCallback, nullptr); + CustomVerificationFunction, nullptr); break; } @@ -2402,13 +2523,6 @@ tsi_result tsi_create_ssl_server_handshaker_factory_with_options( SSL_CTX_set_ex_data(impl->ssl_contexts[i], g_ssl_ctx_ex_crl_provider_index, options->crl_provider.get()); - X509_STORE* cert_store = SSL_CTX_get_cert_store(impl->ssl_contexts[i]); - X509_STORE_set_get_crl(cert_store, GetCrlFromProvider); - X509_STORE_set_check_crl(cert_store, CheckCrlPassthrough); - X509_STORE_set_verify_cb(cert_store, verify_cb); - X509_VERIFY_PARAM* param = X509_STORE_get0_param(cert_store); - X509_VERIFY_PARAM_set_flags( - param, X509_V_FLAG_CRL_CHECK | X509_V_FLAG_CRL_CHECK_ALL); } else if (options->crl_directory != nullptr && strcmp(options->crl_directory, "") != 0) { X509_STORE* cert_store = SSL_CTX_get_cert_store(impl->ssl_contexts[i]); diff --git a/test/core/tsi/BUILD b/test/core/tsi/BUILD index e3c2979ff66..a4854f4dedf 100644 --- a/test/core/tsi/BUILD +++ b/test/core/tsi/BUILD @@ -139,7 +139,10 @@ grpc_cc_test( "//test/core/tsi/test_creds/crl_data/crls:ab06acdd.r0", "//test/core/tsi/test_creds/crl_data/crls:b9322cac.r0", "//test/core/tsi/test_creds/crl_data/crls:current.crl", + "//test/core/tsi/test_creds/crl_data/crls:evil.crl", "//test/core/tsi/test_creds/crl_data/crls:intermediate.crl", + "//test/core/tsi/test_creds/crl_data/crls:invalid_content.crl", + "//test/core/tsi/test_creds/crl_data/crls:invalid_signature.crl", "//test/core/tsi/test_creds/crl_data/crls_missing_intermediate:ab06acdd.r0", "//test/core/tsi/test_creds/crl_data/crls_missing_root:b9322cac.r0", ], diff --git a/test/core/tsi/crl_ssl_transport_security_test.cc b/test/core/tsi/crl_ssl_transport_security_test.cc index 5b13fafbdb7..da873e80956 100644 --- a/test/core/tsi/crl_ssl_transport_security_test.cc +++ b/test/core/tsi/crl_ssl_transport_security_test.cc @@ -68,6 +68,11 @@ const char* kRevokedIntermediateCertPath = const char* kRootCrlPath = "test/core/tsi/test_creds/crl_data/crls/current.crl"; const char* kIntermediateCrlPath = "test/core/tsi/test_creds/crl_data/crls/intermediate.crl"; +const char* kModifiedSignaturePath = + "test/core/tsi/test_creds/crl_data/crls/invalid_signature.crl"; +const char* kModifiedContentPath = + "test/core/tsi/test_creds/crl_data/crls/invalid_content.crl"; +const char* kEvilCrlPath = "test/core/tsi/test_creds/crl_data/crls/evil.crl"; class CrlSslTransportSecurityTest : public testing::TestWithParam { @@ -418,6 +423,51 @@ std::string TestNameSuffix( return "TLS_1_3"; } +TEST_P(CrlSslTransportSecurityTest, CrlProviderModifiedContentCrl) { + std::string root_crl = + grpc_core::testing::GetFileContents(kModifiedContentPath); + std::string intermediate_crl = + grpc_core::testing::GetFileContents(kIntermediateCrlPath); + + absl::StatusOr> + provider = grpc_core::experimental::CreateStaticCrlProvider( + {root_crl, intermediate_crl}); + ASSERT_NE(provider.status(), absl::OkStatus()) << provider.status(); +} + +TEST_P(CrlSslTransportSecurityTest, CrlProviderModifiedSignatureCrl) { + std::string root_crl = + grpc_core::testing::GetFileContents(kModifiedSignaturePath); + std::string intermediate_crl = + grpc_core::testing::GetFileContents(kIntermediateCrlPath); + + absl::StatusOr> + provider = grpc_core::experimental::CreateStaticCrlProvider( + {root_crl, intermediate_crl}); + ASSERT_TRUE(provider.ok()) << provider.status(); + + auto* fixture = new SslTsiTestFixture(kValidKeyPath, kValidCertPath, + kValidKeyPath, kValidCertPath, nullptr, + *provider, false, false, false); + fixture->Run(); +} + +TEST_P(CrlSslTransportSecurityTest, CrlFromBadCa) { + std::string root_crl = grpc_core::testing::GetFileContents(kEvilCrlPath); + std::string intermediate_crl = + grpc_core::testing::GetFileContents(kIntermediateCrlPath); + + absl::StatusOr> + provider = grpc_core::experimental::CreateStaticCrlProvider( + {root_crl, intermediate_crl}); + ASSERT_TRUE(provider.ok()) << provider.status(); + + auto* fixture = new SslTsiTestFixture(kValidKeyPath, kValidCertPath, + kValidKeyPath, kValidCertPath, nullptr, + *provider, false, false, false); + fixture->Run(); +} + // TODO(gtcooke94) Add nullptr issuer test cases - this is not simple to test // the way the code is currently designed - we plan to refactor ways the OpenSSL // callback functions are written to have more easily testable chunks in diff --git a/test/core/tsi/test_creds/crl_data/BUILD b/test/core/tsi/test_creds/crl_data/BUILD index 967fa52845a..8ebab0d3730 100644 --- a/test/core/tsi/test_creds/crl_data/BUILD +++ b/test/core/tsi/test_creds/crl_data/BUILD @@ -26,6 +26,7 @@ exports_files([ "intermediate_ca.key", "intermediate_ca.pem", "evil_ca.pem", + "evil_ca.key", "ca_with_akid.pem", "crl_with_akid.crl", ]) diff --git a/test/core/tsi/test_creds/crl_data/README b/test/core/tsi/test_creds/crl_data/README index e3134548534..f69fd1b75b3 100644 --- a/test/core/tsi/test_creds/crl_data/README +++ b/test/core/tsi/test_creds/crl_data/README @@ -55,6 +55,12 @@ Generate a CA and CRL with an Authority Key Identifier ---------------------------------------------------------------------------- Run `ca_with_akid_gen.sh` from the `test/core/tsi/test_creds/crl_data` directory +Create CRLs with modified signatures and content +---------------------------------------------------------------------------- +Make two copies of `test/core/tsi/test_creds/crl_data/crls/current.crl` named `test/core/tsi/test_creds/crl_data/crls/invalid_content.crl` and `test/core/tsi/test_creds/crl_data/crls/invalid_signature.crl`. +In `invalid_content.crl`, change the first letter on the second line. +In `invalid_signature.crl`, change the last letter before the `=` on the second to last line. + Clean up: --------- diff --git a/test/core/tsi/test_creds/crl_data/crls/BUILD b/test/core/tsi/test_creds/crl_data/crls/BUILD index ecbaa82a9df..e1a36f42c8f 100644 --- a/test/core/tsi/test_creds/crl_data/crls/BUILD +++ b/test/core/tsi/test_creds/crl_data/crls/BUILD @@ -21,4 +21,5 @@ exports_files([ "intermediate.crl", "invalid_signature.crl", "invalid_content.crl", + "evil.crl", ]) diff --git a/test/core/tsi/test_creds/crl_data/crls/evil.crl b/test/core/tsi/test_creds/crl_data/crls/evil.crl new file mode 100644 index 00000000000..ad9118ef095 --- /dev/null +++ b/test/core/tsi/test_creds/crl_data/crls/evil.crl @@ -0,0 +1,12 @@ +-----BEGIN X509 CRL----- +MIIB0TCBugIBATANBgkqhkiG9w0BAQsFADBWMQswCQYDVQQGEwJBVTETMBEGA1UE +CAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRk +MQ8wDQYDVQQDDAZ0ZXN0Y2EXDTI0MDMwNjIxMjE0M1oXDTM0MDMwNDIxMjE0M1qg +MDAuMB8GA1UdIwQYMBaAFNRNe7qb2nAx+OXMM6aMHKZtclpDMAsGA1UdFAQEAgIQ +ADANBgkqhkiG9w0BAQsFAAOCAQEAfynY04pFrcIOUmlKAqchQXlRfdfRHLKmXmRL +L16p//b+Aq6jns8WOJ6DmfCBdy8h+kQwyh1HEB1yxYQGn3OJwR0NRK8riBhyhxkx +akyP1TNMLYPsK/JUBqAvgIfk37oFLKhDO8etYDBndNcNdFs6hryKe40A6eULJXGE +TXY8dTtT++fRX6VbeAaT02d0F+OHhuBEk/WncuGCe1StFEiLau8ZEalB02vv05Wy +H8pn+O4P1oEMg0g/jeMWCqnrJQE3Ut7t2LSLBTgHGTk0cOXyYP2LcO0SVeAbjhtq +qzUSWoxJu98N3y+hqu3FMJA/k0Z0d6PeZ50D3FjbUkT0ZM9f/g== +-----END X509 CRL----- diff --git a/test/core/tsi/test_creds/crl_data/evil_ca.cnf b/test/core/tsi/test_creds/crl_data/evil_ca.cnf new file mode 100644 index 00000000000..5a8bc149120 --- /dev/null +++ b/test/core/tsi/test_creds/crl_data/evil_ca.cnf @@ -0,0 +1,43 @@ +[req] +distinguished_name = req_distinguished_name +req_extensions = v3_req + +[req_distinguished_name] +countryName = Country Name (2 letter code) +countryName_default = AU +stateOrProvinceName = State or Province Name (full name) +stateOrProvinceName_default = Some-State +organizationName = Organization Name (eg, company) +organizationName_default = Internet Widgits Pty Ltd +commonName = Common Name (eg, YOUR name) +commonName_default = testca + +[crl_ext] +authorityKeyIdentifier=keyid:always + +[v3_req] +keyUsage = critical, digitalSignature, keyEncipherment, keyCertSign, cRLSign +basicConstraints = critical, CA:true + +[ca] +default_ca = CA_evil + +[CA_evil] +dir = . +certs = $dir/certs +crl_dir = $dir/crl +new_certs_dir = $dir/newcerts +database = $dir/index.txt +serial = $dir/serial +RANDFILE = $dir/private/.rand +private_key = $dir/evil_ca.key +certificate = $dir/evil_ca.pem +crl = $dir/evil.crl + +# For certificate revocation lists. +crlnumber = $dir/crlnumber +crl = $dir/crl/evil.crl +crl_extensions = crl_ext +default_crl_days = 3650 + +default_md = sha256 diff --git a/test/core/tsi/test_creds/crl_data/evil_ca.key b/test/core/tsi/test_creds/crl_data/evil_ca.key index 41614f98bc4..1682170f307 100644 --- a/test/core/tsi/test_creds/crl_data/evil_ca.key +++ b/test/core/tsi/test_creds/crl_data/evil_ca.key @@ -1,28 +1,28 @@ -----BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC7RwHo8bWaioeI -oqq4qRkiRfqAl/XlaRCyygkMtkjuOy0LA42+LFXXNvDD8eVvVd3615Qopm0XzABd -iz2QiJBZH9qvvmZFg7vG4rbNMCHIN+0YYIOp5tJuyBUVhZ+/f/jZ+LoJeZgTRngQ -tMUmhs7kn4ttT+DC7ZHKhPf5vUokSPG4N2tBx21y2BzRup36q09vfvZeVEe5YxAM -KGWEOcCY/S5vTVeEJCqP2OfMmskIHq2cYWr6ZJzBpdhJXX6rTDWYlCzX49mzPrn6 -povhA/bENv9Gy1OHqPKt+EWEJCaurerkFwF74OG9zp/jCKZJTVkyxnCYjT2rYiDX -gWvNwdeHAgMBAAECggEADyya44Mzj0Y6jXV8tsIA0YLxCrAFZ7q3ydIj9z3ih+cP -PcK3yUPHYCJJUjR3PipWIP03Dy949xd7pMNjpXfjQPgbRz0lWpboxUiDvk7FlfcD -b4O2d12cCbI4Px+uHh1M48B1tnnTOtCYFDvJc6yITARUuZ03cs6UDwrvcB1dygsO -2sZLUOkWQb2DCMq86bxmkHvjuh3gj/CMTJv0Kprlo3YcKNgCwiNygEzlusyIcwpf -dU/SNoWcxY+F0F6wFC0uj75wWqDB6bmfCpY8Bb3Ey7TgWDTWjsB/NQsWbSxZ9o5i -qjQ6WSLKpLLLB/8dXxhk3Nz9tfonavBpLB+4fNpFFQKBgQDi61A3/U88iEo+sxMm -L3i0OS9g/mAnYQ7zYjq42eVyDTfa+eBck1Jmp1KEblfy7Eo3iyApNFoIzFz8va8N -tPNFK/K4mrf1aiFOk0SnvCstW8SBS99hBHXqrMnXrRh+L/OafM4sj88P4RbZxcIs -9RNiDIqcXAPDVU5aHIhs7CFzYwKBgQDTRyOR9PoTQnu0HV0ODDNzmP1eRWrXZ62N -khe9bm0TIG25Q1wsoR6MT5fxZlTe62FH7A5QgEheRtMctr+XGC2H+3N3MUxsTy37 -knPFiDl6Gs5DqKroewiDNbkziMOgctG/z6ORPiGghTRsn6y5dBaMstfvgip8fj5z -ytzgSfiujQKBgHZraOSfK++iDGTmHRMraOlcgm4ysck7LIs08wIurD+1yDVde4m0 -VCdAIJ792qXqS9zqnPED4gx/YfN/pdAYY2/wvG08SM4pAZK45fZHC51TK5xyFPPT -WRoL7BXCvmpz6cPwZ8P3lI5r3/nr6yZ9Cw17EAcDOe+BIC+EfmmhXN+TAoGBAIp0 -oDbSV9+vPen3JDhEfqNOqxvQWgf3haC1EKGvcAOMyNsT7Z/BpodE0cn8ybmcfw/m -/ip7JvHBcC/tAvk9evkWK8D8qZyA9x1aCEx2zVPbpThpnDbmCdoSpt/CzJClLheJ -NyPDl73eDVDyAvs1vGFQAnqOztDu2nZ/huflEfcxAoGAbLUQV5PjqJrsIosEMXsv -qOzQZ5BBEk/jo9zqYSNXWVs0I9Invj5iAYewoM5qn9DFQ3q3O/mPHxF6HT7JHfjn -T8wdOTQk5L1yaaSFsiti3C3AQ2zShT1k6m3V+mf0iWJw878LCURQQFNIHu7zVdXy -4xwQpVw2CN7iufRYN7kOcDo= +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDahYJBV8f013Vm +W+1SiJ/pQFyQCvsCCsy3fUHlUVImzLk7QqSknniehz4XbS12gol9cq4uu18sINFk +bvNNxPiuUDpLn6uguisKwM15h5x8SR0YoYepebs2yFahlL7Qo+vxxTfqamhyKmaE +1d+EvU/l4oAcWp/BQbkInPK2o4S9SJJ7hDlJOWDbYxkn+G2sDwyeWjVGuDbyLl1n +ngp43pd2JcCFnexculaQ9k8yXh6i5wv5SeWfMKgszUzV6SIToDUud4OqXHiz2wmy +kEL+nHb7yjQBjqFJNdsBaJOE3Rq0/VGRGq9R3rrv6CJehU5pRxG/QAd5x3G2A20w +WBkeTxarAgMBAAECggEAGMCZvgwS7sV/G16bVcd7EaFEOt67iwItKTWrgq3A9/sl +mjRU0P7QW+im3GF2Dl//8fFNEKcRwz5eaZl1vt/qaVhWGh3Wg4jC+l9XhwYY8C0Z ++iHF66kJz01HHttp99kxjzvPNyLhfNkXrsFJJdCJ3djXuR58zRfEPVkF1zFThlsd +9HIEU7vgl0zrtnq8cm1+jTpCd6Cv74INtFsEYqAPWdmR+32z3OqSOSJtY8rMi5Ly +ZxjdHaGNT3k/8eA1yDfYmIpYgWQimzEH3FUZlwVFlfEk9GOsbTK0XDEwjTJn68LO +5FNIpfRB2+HpdS4n6w07SLe4lJ5Mv9cJmAs1k7n/2QKBgQD30OhISkA8XlzgtnIB +aN5A3Fv41SRVAO8PKdWKxJJcpeii72OdoHtIHY8FvGdyuRNhinkYhuVydwwRhIrz +XLvVlpVpywEhesDoLoij0hgO+lNfxbEiH18S6Q5LuC5932UwLgbBezHklYYeNpRV +WYWtRYeucNFPLi5DA4H+/y/faQKBgQDhvPBeZCmidMyYb6W85nHr9Y4IPLTqx/On +wCEQwsiMnXq6nM7bacKhJKs4wy+3KO9ObNc+Fd/BnXU/JC/l5tJpwMjW00g8byKu +cZlOglaOADtlxueXCBLV27V8L7Cx5iOicV2ouOaJjMps4k/fWIOIK1XZHj4txJXj +C/7/RvMW8wKBgDU8tNnq0Zfmca94ok85Nx3Z+QwgxdhZBgJM62oPRp4OqkZuhQj0 +0+cvKm2CBvs3VTmMJO2m9R29A2O0BKG5V0TQP7LlgI2vsEdwz7vZw39cOZMGhkId +WTBXztFndN1no3ZRPPRNwe8oTBKriPw46iXKHRbVd7G56whMdZ3RNniZAoGBAL1x +qugMd0R3cRyc1iLp4sF7mm8fQ1Wl6L3nZ9iBH32iy9TAtHk/EK/b7jX82JaGLA9N +GHZqNRZv5m3PGMOAKyXFPMfNGNpfCmQLwfU5PRp+51pKyyDdDbGcaXqHK2qhEVbK +fSeTxSW6mkc2xoFR71DfzXQhBV2zlXauIppqGKgxAoGAMh6fd4tFBa2lpGsfK1Kp +bXKmSMe0P9aV+JrjWHACXcS0n+v/LvNLZrddp+RF4t2M5U7ZujHJEJAHafCo56qk +MkV2WseGb7CBlFQjLKYVXJsiNCf5O5ME+cckT2bK1aX0/h8dOtXirwuQ4OVw2c+n +7/MDH+S5WiEX51Cu4zdVMis= -----END PRIVATE KEY----- diff --git a/test/core/tsi/test_creds/crl_data/evil_ca.pem b/test/core/tsi/test_creds/crl_data/evil_ca.pem index 6708690b1db..77a0f22e258 100644 --- a/test/core/tsi/test_creds/crl_data/evil_ca.pem +++ b/test/core/tsi/test_creds/crl_data/evil_ca.pem @@ -1,21 +1,21 @@ -----BEGIN CERTIFICATE----- -MIIDeTCCAmGgAwIBAgIUULA9nt1NB3W1i4RevrKeRQQLkaIwDQYJKoZIhvcNAQEL +MIIDeTCCAmGgAwIBAgIURqastxiKmyjvJwoaXfh8hA1mccIwDQYJKoZIhvcNAQEL BQAwVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGdGVzdGNhMB4XDTI0 -MDEyMjIxNDAyMFoXDTM0MDExOTIxNDAyMFowVjELMAkGA1UEBhMCQVUxEzARBgNV +MDMwNjE5NDcwMVoXDTM0MDMwNDE5NDcwMVowVjELMAkGA1UEBhMCQVUxEzARBgNV BAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0 ZDEPMA0GA1UEAwwGdGVzdGNhMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC -AQEAu0cB6PG1moqHiKKquKkZIkX6gJf15WkQssoJDLZI7jstCwONvixV1zbww/Hl -b1Xd+teUKKZtF8wAXYs9kIiQWR/ar75mRYO7xuK2zTAhyDftGGCDqebSbsgVFYWf -v3/42fi6CXmYE0Z4ELTFJobO5J+LbU/gwu2RyoT3+b1KJEjxuDdrQcdtctgc0bqd -+qtPb372XlRHuWMQDChlhDnAmP0ub01XhCQqj9jnzJrJCB6tnGFq+mScwaXYSV1+ -q0w1mJQs1+PZsz65+qaL4QP2xDb/RstTh6jyrfhFhCQmrq3q5BcBe+Dhvc6f4wim -SU1ZMsZwmI09q2Ig14FrzcHXhwIDAQABoz8wPTAMBgNVHRMEBTADAQH/MA4GA1Ud -DwEB/wQEAwIBBjAdBgNVHQ4EFgQUjcQvfJ6kAUgljgToPpQ0DmCW0Q8wDQYJKoZI -hvcNAQELBQADggEBALLNhOYqlhOcCsTD1SPfm9MAjfpV1EjSjDCpIfwCk5gI2CUX -g7MyUzn2gQJUiYx74BKmjv6W/sLzNxqR0wZQUr4d/7HX+Lm0xCCYdIUELEM8lZ30 -maBJ599cQnLXDB1ZFEekj3DMM6jL7OQnBaDs5jW4GcDcuwd5cgXfgIaZVjBVJ11Y -CFAhIuh5CM8xhqxWYWY+h0VLU64s8WCNrBEy1OU5KpQRfpd4cvpoWn7E1SfhK1Iq -Bp+1k4oDBpGGw4NLXI3i1aU8x1+KoXxNRg5dOED0OLgppvaWB2yIpqBlcZDaNpq4 -P+WFGBiSUpWU5yYwCDvQAgTWtWkmyflVwslHaGs= +AQEAlDQ88qcz8a9SdLslQrRsN6EJkgWS1dQZ9mzgeGdNyWULlqmOjqP7JZecQSfG +KKA01wkmnzQwaw2HY+kcDw48HKBkjOVVctat4sFg4Du7cwxZPhDTnqxLs1U5poNH +w53gwYd62NDYmGk10J5MbgMmREqPnAVWKHSpNdGErJ9T/AJxlc/QyMKICmt6Iond +TUOWFti3e/K9fqTi9d9Oa6u7hxRky2ZWn3t1NE/p1UMDFcG3Ugn9YkGB6ZPbzno2 +vNWwN3UmV2HOW2QzVmghUm8KlkvaNdRJ5+YvdEAktNS6NNVkoqXo2cfFdQkTtHu/ +OdFCmsIyGBkrpTi4Rq6ObBE+/QIDAQABoz8wPTAMBgNVHRMEBTADAQH/MA4GA1Ud +DwEB/wQEAwIBBjAdBgNVHQ4EFgQUBDQP4CFbiBzHqvAh6TVpA78MeJYwDQYJKoZI +hvcNAQELBQADggEBABwNHIQXyV+8mvvKpC47rUtRvMuFruRmqZb2lET/NiVzazq/ +s3FNNFKTc8DOQzWYhxF5kMSd0+pL7zK7qAkTi+/Gxc7bJpyFvxQZ6FvVgtz2skv1 +8MD3FIcfq3VhbHQnmbp8AY1YGM2uvduSReLEWTz7SIx8bZxDxl8g6K5V71XIWM5X +CHuk7GybN5gemI+WE1a+1wXcL6FVaWCQHrJVT2ZNS1r5rVXOObf8Ubh5gR2kaVY8 +f69OkJ0+XDCOXQw3zmTafnKBtXYdYdT/lMIh8OiseX08W33EiBDczJqFeS/crJZ+ +Tlj+UK1Mtt1NyFu1YV/C7dmGSAZUsdTVM0nY5HA= -----END CERTIFICATE----- diff --git a/test/core/tsi/test_creds/crl_data/evil_ca_gen.sh b/test/core/tsi/test_creds/crl_data/evil_ca_gen.sh index 31dec70057c..d6778fcac38 100644 --- a/test/core/tsi/test_creds/crl_data/evil_ca_gen.sh +++ b/test/core/tsi/test_creds/crl_data/evil_ca_gen.sh @@ -14,5 +14,21 @@ # limitations under the License. # Generates a CA with the same issuer name as the good CA in this directory +rm -rf evil_ca +mkdir evil_ca +cp evil_ca.cnf evil_ca/ +pushd evil_ca || exit +touch index.txt +echo 1 > ./serial +echo 1000 > ./crlnumber + +# Generate the CA with the same subject as the good CA openssl req -x509 -new -newkey rsa:2048 -nodes -keyout evil_ca.key -out evil_ca.pem \ - -config ca-openssl.cnf -days 3650 -extensions v3_req + -config evil_ca.cnf -days 3650 -extensions v3_req + +# Generate the CRL file: +# ---------------------------------------------------------------------------- +openssl ca -config=evil_ca.cnf -gencrl -out evil.crl -keyfile evil_ca.key -cert evil_ca.pem -crldays 3650 +popd || exit +cp "./evil_ca/evil.crl" ./crls/ +rm -rf evil_ca \ No newline at end of file From 57ff0b890fbc95672c198d6840f863ef42e683b5 Mon Sep 17 00:00:00 2001 From: Eugene Ostroukhov Date: Tue, 12 Mar 2024 13:21:54 -0700 Subject: [PATCH 33/52] [xds] Support for multiple servers in the bootstrap.json (#36021) Closes #36021 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/36021 from eugeneo:tasks/multiple-servers-bootstrap 8ebf7e1c064f2d5215f967d47cc1493cbbf12049 PiperOrigin-RevId: 615150672 --- src/core/ext/xds/xds_bootstrap.h | 6 +- src/core/ext/xds/xds_bootstrap_grpc.cc | 35 ++++---- src/core/ext/xds/xds_bootstrap_grpc.h | 19 ++++- src/core/ext/xds/xds_client.cc | 5 +- test/core/memory_usage/memory_usage_test.cc | 2 +- test/core/xds/xds_bootstrap_test.cc | 85 ++++++++++--------- test/core/xds/xds_client_fuzzer.cc | 8 +- test/core/xds/xds_client_test.cc | 50 ++++++----- .../xds/xds_cluster_resource_type_test.cc | 5 +- test/core/xds/xds_common_types_test.cc | 3 +- .../xds/xds_endpoint_resource_type_test.cc | 3 +- .../xds/xds_listener_resource_type_test.cc | 3 +- .../xds_route_config_resource_type_test.cc | 3 +- .../end2end/xds/xds_cluster_end2end_test.cc | 2 +- test/cpp/end2end/xds/xds_core_end2end_test.cc | 28 +++--- test/cpp/end2end/xds/xds_csds_end2end_test.cc | 2 +- test/cpp/end2end/xds/xds_end2end_test.cc | 9 +- test/cpp/end2end/xds/xds_end2end_test_lib.cc | 10 +-- test/cpp/end2end/xds/xds_end2end_test_lib.h | 7 +- .../end2end/xds/xds_routing_end2end_test.cc | 2 +- test/cpp/end2end/xds/xds_utils.cc | 25 +++--- test/cpp/end2end/xds/xds_utils.h | 11 +-- 22 files changed, 181 insertions(+), 142 deletions(-) diff --git a/src/core/ext/xds/xds_bootstrap.h b/src/core/ext/xds/xds_bootstrap.h index 8a1a8214f65..5754319ccfd 100644 --- a/src/core/ext/xds/xds_bootstrap.h +++ b/src/core/ext/xds/xds_bootstrap.h @@ -65,16 +65,14 @@ class XdsBootstrap { public: virtual ~Authority() = default; - virtual const XdsServer* server() const = 0; + virtual std::vector servers() const = 0; }; virtual ~XdsBootstrap() = default; virtual std::string ToString() const = 0; - // TODO(roth): We currently support only one server. Fix this when we - // add support for fallback for the xds channel. - virtual const XdsServer& server() const = 0; + virtual std::vector servers() const = 0; // Returns the node information, or null if not present in the bootstrap // config. diff --git a/src/core/ext/xds/xds_bootstrap_grpc.cc b/src/core/ext/xds/xds_bootstrap_grpc.cc index dd2b3da3c87..553866d667b 100644 --- a/src/core/ext/xds/xds_bootstrap_grpc.cc +++ b/src/core/ext/xds/xds_bootstrap_grpc.cc @@ -14,17 +14,25 @@ // limitations under the License. // -#include - #include "src/core/ext/xds/xds_bootstrap_grpc.h" +#include +#include #include #include #include +#include #include #include +#include "src/core/lib/config/core_configuration.h" +#include "src/core/lib/gprpp/ref_counted_ptr.h" +#include "src/core/lib/json/json.h" +#include "src/core/lib/json/json_object_loader.h" +#include "src/core/lib/json/json_reader.h" +#include "src/core/lib/json/json_writer.h" +#include "src/core/lib/security/credentials/channel_creds_registry.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/match.h" @@ -34,16 +42,6 @@ #include "absl/strings/string_view.h" #include "absl/types/optional.h" -#include - -#include "src/core/lib/config/core_configuration.h" -#include "src/core/lib/gprpp/ref_counted_ptr.h" -#include "src/core/lib/json/json.h" -#include "src/core/lib/json/json_object_loader.h" -#include "src/core/lib/json/json_reader.h" -#include "src/core/lib/json/json_writer.h" -#include "src/core/lib/security/credentials/channel_creds_registry.h" - namespace grpc_core { // @@ -330,11 +328,14 @@ std::string GrpcXdsBootstrap::ToString() const { parts.push_back( absl::StrFormat(" client_listener_resource_name_template=\"%s\",\n", entry.second.client_listener_resource_name_template())); - if (entry.second.server() != nullptr) { - parts.push_back(absl::StrFormat( - " servers=[\n%s\n],\n", - JsonDump(static_cast(entry.second.server()) - ->ToJson()))); + std::vector server_jsons; + for (const XdsServer* server : entry.second.servers()) { + server_jsons.emplace_back( + JsonDump(static_cast(server)->ToJson())); + } + if (!server_jsons.empty()) { + parts.push_back(absl::StrFormat(" servers=[\n%s\n],\n", + absl::StrJoin(server_jsons, ",\n"))); } parts.push_back(" },\n"); } diff --git a/src/core/ext/xds/xds_bootstrap_grpc.h b/src/core/ext/xds/xds_bootstrap_grpc.h index e3506833c65..13c6c8ac083 100644 --- a/src/core/ext/xds/xds_bootstrap_grpc.h +++ b/src/core/ext/xds/xds_bootstrap_grpc.h @@ -104,8 +104,13 @@ class GrpcXdsBootstrap : public XdsBootstrap { class GrpcAuthority : public Authority { public: - const XdsServer* server() const override { - return servers_.empty() ? nullptr : &servers_[0]; + std::vector servers() const override { + std::vector servers; + servers.reserve(servers_.size()); + for (const auto& server : servers_) { + servers.emplace_back(&server); + } + return servers; } const std::string& client_listener_resource_name_template() const { @@ -129,7 +134,15 @@ class GrpcXdsBootstrap : public XdsBootstrap { std::string ToString() const override; - const XdsServer& server() const override { return servers_[0]; } + std::vector servers() const override { + std::vector servers; + servers.reserve(servers_.size()); + for (const auto& server : servers_) { + servers.emplace_back(&server); + } + return servers; + } + const Node* node() const override { return node_.has_value() ? &*node_ : nullptr; } diff --git a/src/core/ext/xds/xds_client.cc b/src/core/ext/xds/xds_client.cc index 08493a414ac..c9c65ab3e51 100644 --- a/src/core/ext/xds/xds_client.cc +++ b/src/core/ext/xds/xds_client.cc @@ -1584,9 +1584,10 @@ void XdsClient::WatchResource(const XdsResourceType* type, "\" not present in bootstrap config"))); return; } - xds_server = authority->server(); + xds_server = + authority->servers().empty() ? nullptr : authority->servers().front(); } - if (xds_server == nullptr) xds_server = &bootstrap_->server(); + if (xds_server == nullptr) xds_server = bootstrap_->servers().front(); { MutexLock lock(&mu_); MaybeRegisterResourceTypeLocked(type); diff --git a/test/core/memory_usage/memory_usage_test.cc b/test/core/memory_usage/memory_usage_test.cc index 5b178b6d7f4..1ca0fc4aca5 100644 --- a/test/core/memory_usage/memory_usage_test.cc +++ b/test/core/memory_usage/memory_usage_test.cc @@ -210,7 +210,7 @@ XdsServer StartXdsServerAndConfigureBootstrap( // Generate xDS bootstrap and set the env var. std::string bootstrap = grpc::testing::XdsBootstrapBuilder() - .SetDefaultServer(absl::StrCat("localhost:", xds_server_port)) + .SetServers({absl::StrCat("localhost:", xds_server_port)}) .SetXdsChannelCredentials("insecure") .Build(); grpc_core::SetEnv("GRPC_XDS_BOOTSTRAP_CONFIG", bootstrap); diff --git a/test/core/xds/xds_bootstrap_test.cc b/test/core/xds/xds_bootstrap_test.cc index 749d44769e8..0749a9006e3 100644 --- a/test/core/xds/xds_bootstrap_test.cc +++ b/test/core/xds/xds_bootstrap_test.cc @@ -55,12 +55,30 @@ namespace grpc_core { namespace testing { namespace { +MATCHER_P2(EqXdsServer, name, creds_config_type, "equals XdsServer") { + auto* server = static_cast(arg); + if (!::testing::ExplainMatchResult(::testing::Ne(nullptr), server, + result_listener)) { + return false; + } + bool ok = ::testing::ExplainMatchResult(name, server->server_uri(), + result_listener); + auto creds_config = server->channel_creds_config(); + if (!::testing::ExplainMatchResult(::testing::Ne(nullptr), creds_config, + result_listener)) { + return false; + } + ok |= ::testing::ExplainMatchResult(creds_config_type, creds_config->type(), + result_listener); + return ok; +} + TEST(XdsBootstrapTest, Basic) { const char* json_str = "{" " \"xds_servers\": [" " {" - " \"server_uri\": \"fake:///lb\"," + " \"server_uri\": \"fake:///lb1\"," " \"channel_creds\": [" " {" " \"type\": \"fake\"," @@ -70,14 +88,11 @@ TEST(XdsBootstrapTest, Basic) { " \"ignore\": 0" " }," " {" - " \"server_uri\": \"ignored\"," + " \"server_uri\": \"fake:///lb2\"," " \"channel_creds\": [" " {" - " \"type\": \"ignored\"," + " \"type\": \"fake\"," " \"ignore\": 0" - " }," - " {" - " \"type\": \"fake\"" " }" " ]," " \"ignore\": 0" @@ -97,6 +112,15 @@ TEST(XdsBootstrapTest, Basic) { " }" " ]," " \"server_features\": [\"xds_v3\"]" + " }," + " {" + " \"server_uri\": \"fake:///xds_server2\"," + " \"channel_creds\": [" + " {" + " \"type\": \"fake\"" + " }" + " ]," + " \"server_features\": [\"xds_v3\"]" " }" " ]" " }," @@ -106,7 +130,7 @@ TEST(XdsBootstrapTest, Basic) { "server/%s\"," " \"xds_servers\": [" " {" - " \"server_uri\": \"fake:///xds_server2\"," + " \"server_uri\": \"fake:///xds_server3\"," " \"channel_creds\": [" " {" " \"type\": \"fake\"" @@ -138,11 +162,9 @@ TEST(XdsBootstrapTest, Basic) { auto bootstrap_or = GrpcXdsBootstrap::Create(json_str); ASSERT_TRUE(bootstrap_or.ok()) << bootstrap_or.status(); auto bootstrap = std::move(*bootstrap_or); - auto* server = - &static_cast(bootstrap->server()); - EXPECT_EQ(server->server_uri(), "fake:///lb"); - ASSERT_NE(server->channel_creds_config(), nullptr); - EXPECT_EQ(server->channel_creds_config()->type(), "fake"); + EXPECT_THAT(bootstrap->servers(), + ::testing::ElementsAre(EqXdsServer("fake:///lb1", "fake"), + EqXdsServer("fake:///lb2", "fake"))); EXPECT_EQ(bootstrap->authorities().size(), 2); auto* authority = static_cast( bootstrap->LookupAuthority("xds.example.com")); @@ -150,24 +172,18 @@ TEST(XdsBootstrapTest, Basic) { EXPECT_EQ(authority->client_listener_resource_name_template(), "xdstp://xds.example.com/envoy.config.listener.v3.Listener/grpc/" "server/%s"); - server = - static_cast(authority->server()); - ASSERT_NE(server, nullptr); - EXPECT_EQ(server->server_uri(), "fake:///xds_server"); - ASSERT_NE(server->channel_creds_config(), nullptr); - EXPECT_EQ(server->channel_creds_config()->type(), "fake"); + EXPECT_THAT( + authority->servers(), + ::testing::ElementsAre(EqXdsServer("fake:///xds_server", "fake"), + EqXdsServer("fake:///xds_server2", "fake"))); authority = static_cast( bootstrap->LookupAuthority("xds.example2.com")); ASSERT_NE(authority, nullptr); EXPECT_EQ(authority->client_listener_resource_name_template(), "xdstp://xds.example2.com/envoy.config.listener.v3.Listener/grpc/" "server/%s"); - server = - static_cast(authority->server()); - ASSERT_NE(server, nullptr); - EXPECT_EQ(server->server_uri(), "fake:///xds_server2"); - ASSERT_NE(server->channel_creds_config(), nullptr); - EXPECT_EQ(server->channel_creds_config()->type(), "fake"); + EXPECT_THAT(authority->servers(), ::testing::ElementsAre(EqXdsServer( + "fake:///xds_server3", "fake"))); ASSERT_NE(bootstrap->node(), nullptr); EXPECT_EQ(bootstrap->node()->id(), "foo"); EXPECT_EQ(bootstrap->node()->cluster(), "bar"); @@ -203,11 +219,8 @@ TEST(XdsBootstrapTest, ValidWithoutNode) { auto bootstrap_or = GrpcXdsBootstrap::Create(json_str); ASSERT_TRUE(bootstrap_or.ok()) << bootstrap_or.status(); auto bootstrap = std::move(*bootstrap_or); - auto* server = - &static_cast(bootstrap->server()); - EXPECT_EQ(server->server_uri(), "fake:///lb"); - ASSERT_NE(server->channel_creds_config(), nullptr); - EXPECT_EQ(server->channel_creds_config()->type(), "fake"); + EXPECT_THAT(bootstrap->servers(), + ::testing::ElementsAre(EqXdsServer("fake:///lb", "fake"))); EXPECT_EQ(bootstrap->node(), nullptr); } @@ -224,11 +237,8 @@ TEST(XdsBootstrapTest, InsecureCreds) { auto bootstrap_or = GrpcXdsBootstrap::Create(json_str); ASSERT_TRUE(bootstrap_or.ok()) << bootstrap_or.status(); auto bootstrap = std::move(*bootstrap_or); - auto* server = - &static_cast(bootstrap->server()); - EXPECT_EQ(server->server_uri(), "fake:///lb"); - ASSERT_NE(server->channel_creds_config(), nullptr); - EXPECT_EQ(server->channel_creds_config()->type(), "insecure"); + EXPECT_THAT(bootstrap->servers(), + ::testing::ElementsAre(EqXdsServer("fake:///lb", "insecure"))); EXPECT_EQ(bootstrap->node(), nullptr); } @@ -261,11 +271,8 @@ TEST(XdsBootstrapTest, GoogleDefaultCreds) { auto bootstrap_or = GrpcXdsBootstrap::Create(json_str); ASSERT_TRUE(bootstrap_or.ok()) << bootstrap_or.status(); auto bootstrap = std::move(*bootstrap_or); - auto* server = - &static_cast(bootstrap->server()); - EXPECT_EQ(server->server_uri(), "fake:///lb"); - ASSERT_NE(server->channel_creds_config(), nullptr); - EXPECT_EQ(server->channel_creds_config()->type(), "google_default"); + EXPECT_THAT(bootstrap->servers(), ::testing::ElementsAre(EqXdsServer( + "fake:///lb", "google_default"))); EXPECT_EQ(bootstrap->node(), nullptr); } diff --git a/test/core/xds/xds_client_fuzzer.cc b/test/core/xds/xds_client_fuzzer.cc index 3603e7d0d69..4d6994a1381 100644 --- a/test/core/xds/xds_client_fuzzer.cc +++ b/test/core/xds/xds_client_fuzzer.cc @@ -236,13 +236,15 @@ class Fuzzer { const XdsBootstrap::XdsServer* GetServer(const std::string& authority) { const GrpcXdsBootstrap& bootstrap = static_cast(xds_client_->bootstrap()); - if (authority.empty()) return &bootstrap.server(); + if (authority.empty()) return bootstrap.servers().front(); const auto* authority_entry = static_cast( bootstrap.LookupAuthority(authority)); if (authority_entry == nullptr) return nullptr; - if (authority_entry->server() != nullptr) return authority_entry->server(); - return &bootstrap.server(); + if (!authority_entry->servers().empty()) { + return authority_entry->servers().front(); + } + return bootstrap.servers().front(); } void TriggerConnectionFailure(const std::string& authority, diff --git a/test/core/xds/xds_client_test.cc b/test/core/xds/xds_client_test.cc index 7a699bcfcb0..c6365c03e73 100644 --- a/test/core/xds/xds_client_test.cc +++ b/test/core/xds/xds_client_test.cc @@ -20,6 +20,12 @@ #include "src/core/ext/xds/xds_client.h" +#include +#include +#include +#include +#include +#include #include #include @@ -27,23 +33,10 @@ #include #include #include +#include -#include -#include - -#include "absl/strings/str_cat.h" -#include "absl/time/time.h" -#include "absl/types/optional.h" -#include "absl/types/variant.h" #include "gmock/gmock.h" #include "gtest/gtest.h" -#include "upb/reflection/def.h" - -#include -#include -#include -#include - #include "src/core/ext/xds/xds_bootstrap.h" #include "src/core/ext/xds/xds_resource_type_impl.h" #include "src/core/lib/event_engine/default_event_engine.h" @@ -59,6 +52,11 @@ #include "test/core/util/scoped_env_var.h" #include "test/core/util/test_config.h" #include "test/core/xds/xds_transport_fake.h" +#include "absl/strings/str_cat.h" +#include "absl/time/time.h" +#include "absl/types/optional.h" +#include "absl/types/variant.h" +#include "upb/reflection/def.h" // IWYU pragma: no_include // IWYU pragma: no_include @@ -149,8 +147,12 @@ class XdsClientTest : public ::testing::Test { class FakeAuthority : public Authority { public: - const XdsServer* server() const override { - return server_.has_value() ? &*server_ : nullptr; + std::vector servers() const override { + if (server_.has_value()) { + return {&*server_}; + } else { + return {}; + }; } void set_server(absl::optional server) { @@ -194,7 +196,10 @@ class XdsClientTest : public ::testing::Test { std::string ToString() const override { return ""; } - const XdsServer& server() const override { return server_; } + std::vector servers() const override { + return {&server_}; + } + const Node* node() const override { return node_.has_value() ? &*node_ : nullptr; } @@ -659,7 +664,8 @@ class XdsClientTest : public ::testing::Test { RefCountedPtr WaitForAdsStream( absl::Duration timeout = absl::Seconds(5)) { - return WaitForAdsStream(xds_client_->bootstrap().server(), timeout); + return WaitForAdsStream(*xds_client_->bootstrap().servers().front(), + timeout); } // Gets the latest request sent to the fake xDS server. @@ -1809,7 +1815,7 @@ TEST_F(XdsClientTest, ConnectionFails) { /*resource_names=*/{"foo1"}); CheckRequestNode(*request); // Should be present on the first request. // Transport reports connection failure. - TriggerConnectionFailure(xds_client_->bootstrap().server(), + TriggerConnectionFailure(*xds_client_->bootstrap().servers().front(), absl::UnavailableError("connection failed")); // XdsClient should report an error to the watcher. auto error = watcher->WaitForNextError(); @@ -2364,7 +2370,7 @@ TEST_F(XdsClientTest, Federation) { // Watcher should initially not see any resource reported. EXPECT_FALSE(watcher->HasEvent()); // XdsClient should have created an ADS stream to the top-level xDS server. - auto stream = WaitForAdsStream(xds_client_->bootstrap().server()); + auto stream = WaitForAdsStream(*xds_client_->bootstrap().servers().front()); ASSERT_TRUE(stream != nullptr); // XdsClient should have sent a subscription request on the ADS stream. auto request = WaitForRequest(stream.get()); @@ -2450,7 +2456,7 @@ TEST_F(XdsClientTest, FederationAuthorityDefaultsToTopLevelXdsServer) { // Watcher should initially not see any resource reported. EXPECT_FALSE(watcher->HasEvent()); // XdsClient should have created an ADS stream to the top-level xDS server. - auto stream = WaitForAdsStream(xds_client_->bootstrap().server()); + auto stream = WaitForAdsStream(*xds_client_->bootstrap().servers().front()); ASSERT_TRUE(stream != nullptr); // XdsClient should have sent a subscription request on the ADS stream. auto request = WaitForRequest(stream.get()); @@ -2620,7 +2626,7 @@ TEST_F(XdsClientTest, FederationChannelFailureReportedToWatchers) { // Watcher should initially not see any resource reported. EXPECT_FALSE(watcher->HasEvent()); // XdsClient should have created an ADS stream to the top-level xDS server. - auto stream = WaitForAdsStream(xds_client_->bootstrap().server()); + auto stream = WaitForAdsStream(*xds_client_->bootstrap().servers().front()); ASSERT_TRUE(stream != nullptr); // XdsClient should have sent a subscription request on the ADS stream. auto request = WaitForRequest(stream.get()); diff --git a/test/core/xds/xds_cluster_resource_type_test.cc b/test/core/xds/xds_cluster_resource_type_test.cc index af8795e00fc..a30dd8bc253 100644 --- a/test/core/xds/xds_cluster_resource_type_test.cc +++ b/test/core/xds/xds_cluster_resource_type_test.cc @@ -85,7 +85,8 @@ class XdsClusterTest : public ::testing::Test { protected: XdsClusterTest() : xds_client_(MakeXdsClient()), - decode_context_{xds_client_.get(), xds_client_->bootstrap().server(), + decode_context_{xds_client_.get(), + *xds_client_->bootstrap().servers().front(), &xds_cluster_resource_type_test_trace, upb_def_pool_.ptr(), upb_arena_.ptr()} {} @@ -1106,7 +1107,7 @@ TEST_F(LrsTest, Valid) { static_cast(**decode_result.resource); ASSERT_TRUE(resource.lrs_load_reporting_server.has_value()); EXPECT_EQ(*resource.lrs_load_reporting_server, - xds_client_->bootstrap().server()); + *xds_client_->bootstrap().servers().front()); } TEST_F(LrsTest, NotSelfConfigSource) { diff --git a/test/core/xds/xds_common_types_test.cc b/test/core/xds/xds_common_types_test.cc index 4eaf702092f..ab049d954d6 100644 --- a/test/core/xds/xds_common_types_test.cc +++ b/test/core/xds/xds_common_types_test.cc @@ -72,7 +72,8 @@ class XdsCommonTypesTest : public ::testing::Test { protected: XdsCommonTypesTest() : xds_client_(MakeXdsClient()), - decode_context_{xds_client_.get(), xds_client_->bootstrap().server(), + decode_context_{xds_client_.get(), + *xds_client_->bootstrap().servers().front(), &xds_common_types_test_trace, upb_def_pool_.ptr(), upb_arena_.ptr()} {} diff --git a/test/core/xds/xds_endpoint_resource_type_test.cc b/test/core/xds/xds_endpoint_resource_type_test.cc index 08e95f9164f..8538dcd15c0 100644 --- a/test/core/xds/xds_endpoint_resource_type_test.cc +++ b/test/core/xds/xds_endpoint_resource_type_test.cc @@ -71,7 +71,8 @@ class XdsEndpointTest : public ::testing::Test { protected: XdsEndpointTest() : xds_client_(MakeXdsClient()), - decode_context_{xds_client_.get(), xds_client_->bootstrap().server(), + decode_context_{xds_client_.get(), + *xds_client_->bootstrap().servers().front(), &xds_endpoint_resource_type_test_trace, upb_def_pool_.ptr(), upb_arena_.ptr()} {} diff --git a/test/core/xds/xds_listener_resource_type_test.cc b/test/core/xds/xds_listener_resource_type_test.cc index 60f3996d7cb..eac9cb55df7 100644 --- a/test/core/xds/xds_listener_resource_type_test.cc +++ b/test/core/xds/xds_listener_resource_type_test.cc @@ -85,7 +85,8 @@ class XdsListenerTest : public ::testing::Test { protected: XdsListenerTest() : xds_client_(MakeXdsClient()), - decode_context_{xds_client_.get(), xds_client_->bootstrap().server(), + decode_context_{xds_client_.get(), + *xds_client_->bootstrap().servers().front(), &xds_listener_resource_type_test_trace, upb_def_pool_.ptr(), upb_arena_.ptr()} {} diff --git a/test/core/xds/xds_route_config_resource_type_test.cc b/test/core/xds/xds_route_config_resource_type_test.cc index 013b69b0b3e..8703efdfcb2 100644 --- a/test/core/xds/xds_route_config_resource_type_test.cc +++ b/test/core/xds/xds_route_config_resource_type_test.cc @@ -83,7 +83,8 @@ class XdsRouteConfigTest : public ::testing::Test { protected: XdsRouteConfigTest() : xds_client_(MakeXdsClient()), - decode_context_{xds_client_.get(), xds_client_->bootstrap().server(), + decode_context_{xds_client_.get(), + *xds_client_->bootstrap().servers().front(), &xds_route_config_resource_type_test_trace, upb_def_pool_.ptr(), upb_arena_.ptr()} {} diff --git a/test/cpp/end2end/xds/xds_cluster_end2end_test.cc b/test/cpp/end2end/xds/xds_cluster_end2end_test.cc index 2bef38e84b8..7322f06c9f6 100644 --- a/test/cpp/end2end/xds/xds_cluster_end2end_test.cc +++ b/test/cpp/end2end/xds/xds_cluster_end2end_test.cc @@ -411,7 +411,7 @@ TEST_P(CdsDeletionTest, ClusterDeleted) { // Tests that we ignore Cluster deletions if configured to do so. TEST_P(CdsDeletionTest, ClusterDeletionIgnored) { - InitClient(XdsBootstrapBuilder().SetIgnoreResourceDeletion()); + InitClient(MakeBootstrapBuilder().SetIgnoreResourceDeletion()); CreateAndStartBackends(2); // Bring up client pointing to backend 0 and wait for it to connect. EdsResourceArgs args({{"locality0", CreateEndpointsForBackends(0, 1)}}); diff --git a/test/cpp/end2end/xds/xds_core_end2end_test.cc b/test/cpp/end2end/xds/xds_core_end2end_test.cc index a0afbaaec82..752ed01fe55 100644 --- a/test/cpp/end2end/xds/xds_core_end2end_test.cc +++ b/test/cpp/end2end/xds/xds_core_end2end_test.cc @@ -324,7 +324,7 @@ TEST_P(GlobalXdsClientTest, InvalidListenerStillExistsIfPreviouslyCached) { class TimeoutTest : public XdsEnd2endTest { protected: void SetUp() override { - InitClient(XdsBootstrapBuilder(), /*lb_expected_authority=*/"", + InitClient(MakeBootstrapBuilder(), /*lb_expected_authority=*/"", /*xds_resource_does_not_exist_timeout_ms=*/2000); } }; @@ -644,7 +644,7 @@ TEST_P(XdsFederationTest, FederationTargetNoAuthorityWithResourceTemplate) { const char* kNewClusterName = "xdstp://xds.example.com/envoy.config.cluster.v3.Cluster/" "new_cluster_name"; - XdsBootstrapBuilder builder; + XdsBootstrapBuilder builder = MakeBootstrapBuilder(); builder.SetClientDefaultListenerResourceNameTemplate(kNewListenerTemplate); builder.AddAuthority( kAuthority, absl::StrCat("localhost:", authority_balancer_->port()), @@ -698,7 +698,7 @@ TEST_P(XdsFederationTest, FederationTargetAuthorityDefaultResourceTemplate) { const char* kNewClusterName = "xdstp://xds.example.com/envoy.config.cluster.v3.Cluster/" "cluster_name"; - XdsBootstrapBuilder builder; + XdsBootstrapBuilder builder = MakeBootstrapBuilder(); builder.AddAuthority(kAuthority, absl::StrCat("localhost:", authority_balancer_->port())); InitClient(builder); @@ -767,7 +767,7 @@ TEST_P(XdsFederationTest, FederationTargetAuthorityWithResourceTemplate) { const char* kNewClusterName = "xdstp://xds.example.com/envoy.config.cluster.v3.Cluster/" "cluster_name"; - XdsBootstrapBuilder builder; + XdsBootstrapBuilder builder = MakeBootstrapBuilder(); builder.AddAuthority(kAuthority, absl::StrCat("localhost:", authority_balancer_->port()), kNewListenerTemplate); @@ -823,7 +823,7 @@ TEST_P(XdsFederationTest, TargetUriAuthorityUnknown) { const char* kNewListenerTemplate = "xdstp://xds.example.com/envoy.config.listener.v3.Listener/" "client/%s?psm_project_id=1234"; - XdsBootstrapBuilder builder; + XdsBootstrapBuilder builder = MakeBootstrapBuilder(); builder.AddAuthority( kAuthority, absl::StrCat("localhost:", grpc_pick_unused_port_or_die()), kNewListenerTemplate); @@ -854,7 +854,7 @@ TEST_P(XdsFederationTest, RdsResourceNameAuthorityUnknown) { const char* kNewRouteConfigName = "xdstp://xds.unknown.com/envoy.config.route.v3.RouteConfiguration/" "new_route_config_name"; - XdsBootstrapBuilder builder; + XdsBootstrapBuilder builder = MakeBootstrapBuilder(); builder.AddAuthority(kAuthority, absl::StrCat("localhost:", authority_balancer_->port()), kNewListenerTemplate); @@ -900,7 +900,7 @@ TEST_P(XdsFederationTest, CdsResourceNameAuthorityUnknown) { const char* kNewClusterName = "xdstp://xds.unknown.com/envoy.config.cluster.v3.Cluster/" "cluster_name"; - XdsBootstrapBuilder builder; + XdsBootstrapBuilder builder = MakeBootstrapBuilder(); builder.AddAuthority(kAuthority, absl::StrCat("localhost:", authority_balancer_->port()), kNewListenerTemplate); @@ -952,7 +952,7 @@ TEST_P(XdsFederationTest, EdsResourceNameAuthorityUnknown) { const char* kNewClusterName = "xdstp://xds.example.com/envoy.config.cluster.v3.Cluster/" "cluster_name"; - XdsBootstrapBuilder builder; + XdsBootstrapBuilder builder = MakeBootstrapBuilder(); builder.AddAuthority(kAuthority, absl::StrCat("localhost:", authority_balancer_->port()), kNewListenerTemplate); @@ -1019,7 +1019,7 @@ TEST_P(XdsFederationTest, FederationServer) { const char* kNewClusterName = "xdstp://xds.example.com/envoy.config.cluster.v3.Cluster/" "new_cluster_name"; - XdsBootstrapBuilder builder; + XdsBootstrapBuilder builder = MakeBootstrapBuilder(); builder.SetClientDefaultListenerResourceNameTemplate(kNewListenerTemplate); builder.SetServerListenerResourceNameTemplate(kNewServerListenerTemplate); builder.AddAuthority( @@ -1165,7 +1165,7 @@ TEST_P(XdsFederationLoadReportingTest, FederationMultipleLoadReportingTest) { "cluster_name"; const size_t kNumRpcsToDefaultBalancer = 5; const size_t kNumRpcsToAuthorityBalancer = 10; - XdsBootstrapBuilder builder; + XdsBootstrapBuilder builder = MakeBootstrapBuilder(); builder.AddAuthority(kAuthority, absl::StrCat("localhost:", authority_balancer_->port()), kNewListenerTemplate); @@ -1276,11 +1276,11 @@ TEST_P(XdsFederationLoadReportingTest, SameServerInAuthorityAndTopLevel) { const char* kNewEdsServiceName = "xdstp://xds.example.com/envoy.config.endpoint.v3.ClusterLoadAssignment/" "edsservice_name"; - XdsBootstrapBuilder builder; std::string xds_server = absl::StrCat("localhost:", authority_balancer_->port()); + XdsBootstrapBuilder builder; + builder.SetServers({xds_server}); builder.AddAuthority(kAuthority, xds_server); - builder.SetDefaultServer(xds_server); InitClient(builder); CreateAndStartBackends(1); authority_balancer_->lrs_service()->set_send_all_clusters(true); @@ -1349,7 +1349,7 @@ INSTANTIATE_TEST_SUITE_P(XdsTest, SecureNamingTest, // Tests that secure naming check passes if target name is expected. TEST_P(SecureNamingTest, TargetNameIsExpected) { - InitClient(XdsBootstrapBuilder(), /*lb_expected_authority=*/"localhost:%d"); + InitClient(MakeBootstrapBuilder(), /*lb_expected_authority=*/"localhost:%d"); CreateAndStartBackends(4); EdsResourceArgs args({ {"locality0", CreateEndpointsForBackends()}, @@ -1361,7 +1361,7 @@ TEST_P(SecureNamingTest, TargetNameIsExpected) { // Tests that secure naming check fails if target name is unexpected. TEST_P(SecureNamingTest, TargetNameIsUnexpected) { GTEST_FLAG_SET(death_test_style, "threadsafe"); - InitClient(XdsBootstrapBuilder(), + InitClient(MakeBootstrapBuilder(), /*lb_expected_authority=*/"incorrect_server_name"); CreateAndStartBackends(4); EdsResourceArgs args({ diff --git a/test/cpp/end2end/xds/xds_csds_end2end_test.cc b/test/cpp/end2end/xds/xds_csds_end2end_test.cc index cde8148b9c5..2aa0c0645e4 100644 --- a/test/cpp/end2end/xds/xds_csds_end2end_test.cc +++ b/test/cpp/end2end/xds/xds_csds_end2end_test.cc @@ -696,7 +696,7 @@ class CsdsShortAdsTimeoutTest : public ClientStatusDiscoveryServiceTest { protected: void SetUp() override { // Shorten the ADS subscription timeout to speed up the test run. - InitClient(XdsBootstrapBuilder(), /*lb_expected_authority=*/"", + InitClient(absl::nullopt, /*lb_expected_authority=*/"", /*xds_resource_does_not_exist_timeout_ms=*/2000); } }; diff --git a/test/cpp/end2end/xds/xds_end2end_test.cc b/test/cpp/end2end/xds/xds_end2end_test.cc index f48de70c7d1..1415742f198 100644 --- a/test/cpp/end2end/xds/xds_end2end_test.cc +++ b/test/cpp/end2end/xds/xds_end2end_test.cc @@ -279,7 +279,7 @@ FakeCertificateProvider::CertDataMapWrapper* g_fake2_cert_data_map = nullptr; class XdsSecurityTest : public XdsEnd2endTest { protected: void SetUp() override { - XdsBootstrapBuilder builder; + XdsBootstrapBuilder builder = MakeBootstrapBuilder(); builder.AddCertificateProviderPlugin("fake_plugin1", "fake1"); builder.AddCertificateProviderPlugin("fake_plugin2", "fake2"); std::vector fields; @@ -799,7 +799,8 @@ class XdsEnabledServerTest : public XdsEnd2endTest { protected: void SetUp() override {} // No-op -- individual tests do this themselves. - void DoSetUp(XdsBootstrapBuilder builder = XdsBootstrapBuilder()) { + void DoSetUp( + const absl::optional& builder = absl::nullopt) { InitClient(builder); CreateBackends(1, /*xds_enabled=*/true); EdsResourceArgs args({{"locality0", CreateEndpointsForBackends(0, 1)}}); @@ -814,7 +815,7 @@ TEST_P(XdsEnabledServerTest, Basic) { } TEST_P(XdsEnabledServerTest, ListenerDeletionIgnored) { - DoSetUp(XdsBootstrapBuilder().SetIgnoreResourceDeletion()); + DoSetUp(MakeBootstrapBuilder().SetIgnoreResourceDeletion()); backends_[0]->Start(); WaitForBackend(DEBUG_LOCATION, 0); // Check that we ACKed. @@ -908,7 +909,7 @@ TEST_P(XdsEnabledServerTest, ListenerAddressMismatch) { class XdsServerSecurityTest : public XdsEnd2endTest { protected: void SetUp() override { - XdsBootstrapBuilder builder; + XdsBootstrapBuilder builder = MakeBootstrapBuilder(); builder.AddCertificateProviderPlugin("fake_plugin1", "fake1"); builder.AddCertificateProviderPlugin("fake_plugin2", "fake2"); std::vector fields; diff --git a/test/cpp/end2end/xds/xds_end2end_test_lib.cc b/test/cpp/end2end/xds/xds_end2end_test_lib.cc index 6f69e351cfb..121f92328fc 100644 --- a/test/cpp/end2end/xds/xds_end2end_test_lib.cc +++ b/test/cpp/end2end/xds/xds_end2end_test_lib.cc @@ -483,9 +483,12 @@ std::vector XdsEnd2endTest::GetBackendPorts(size_t start_index, return backend_ports; } -void XdsEnd2endTest::InitClient(XdsBootstrapBuilder builder, +void XdsEnd2endTest::InitClient(absl::optional builder, std::string lb_expected_authority, int xds_resource_does_not_exist_timeout_ms) { + if (!builder.has_value()) { + builder = MakeBootstrapBuilder(); + } if (xds_resource_does_not_exist_timeout_ms > 0) { xds_channel_args_to_add_.emplace_back(grpc_channel_arg_integer_create( const_cast(GRPC_ARG_XDS_RESOURCE_DOES_NOT_EXIST_TIMEOUT_MS), @@ -503,10 +506,7 @@ void XdsEnd2endTest::InitClient(XdsBootstrapBuilder builder, } xds_channel_args_.num_args = xds_channel_args_to_add_.size(); xds_channel_args_.args = xds_channel_args_to_add_.data(); - // Initialize XdsClient state. - builder.SetDefaultServer(absl::StrCat("localhost:", balancer_->port()), - /*ignore_if_set=*/true); - bootstrap_ = builder.Build(); + bootstrap_ = builder->Build(); if (GetParam().bootstrap_source() == XdsTestType::kBootstrapFromEnvVar) { grpc_core::SetEnv("GRPC_XDS_BOOTSTRAP_CONFIG", bootstrap_.c_str()); } else if (GetParam().bootstrap_source() == XdsTestType::kBootstrapFromFile) { diff --git a/test/cpp/end2end/xds/xds_end2end_test_lib.h b/test/cpp/end2end/xds/xds_end2end_test_lib.h index 86cc867d008..feff26b608a 100644 --- a/test/cpp/end2end/xds/xds_end2end_test_lib.h +++ b/test/cpp/end2end/xds/xds_end2end_test_lib.h @@ -566,10 +566,15 @@ class XdsEnd2endTest : public ::testing::TestWithParam, // Initializes global state for the client, such as the bootstrap file // and channel args for the XdsClient. Then calls ResetStub(). // All tests must call this exactly once at the start of the test. - void InitClient(XdsBootstrapBuilder builder = XdsBootstrapBuilder(), + void InitClient(absl::optional builder = absl::nullopt, std::string lb_expected_authority = "", int xds_resource_does_not_exist_timeout_ms = 0); + XdsBootstrapBuilder MakeBootstrapBuilder() { + return XdsBootstrapBuilder().SetServers( + {absl::StrCat("localhost:", balancer_->port())}); + } + // Sets channel_, stub_, stub1_, and stub2_. void ResetStub(int failover_timeout_ms = 0, ChannelArguments* args = nullptr); diff --git a/test/cpp/end2end/xds/xds_routing_end2end_test.cc b/test/cpp/end2end/xds/xds_routing_end2end_test.cc index fc05fc17b9c..e9f3fbd0ad7 100644 --- a/test/cpp/end2end/xds/xds_routing_end2end_test.cc +++ b/test/cpp/end2end/xds/xds_routing_end2end_test.cc @@ -115,7 +115,7 @@ TEST_P(LdsDeletionTest, ListenerDeleted) { // Tests that we ignore Listener deletions if configured to do so. TEST_P(LdsDeletionTest, ListenerDeletionIgnored) { - InitClient(XdsBootstrapBuilder().SetIgnoreResourceDeletion()); + InitClient(MakeBootstrapBuilder().SetIgnoreResourceDeletion()); CreateAndStartBackends(2); // Bring up client pointing to backend 0 and wait for it to connect. EdsResourceArgs args({{"locality0", CreateEndpointsForBackends(0, 1)}}); diff --git a/test/cpp/end2end/xds/xds_utils.cc b/test/cpp/end2end/xds/xds_utils.cc index b1d42bdde79..c47302ec66b 100644 --- a/test/cpp/end2end/xds/xds_utils.cc +++ b/test/cpp/end2end/xds/xds_utils.cc @@ -60,7 +60,7 @@ using ::envoy::extensions::filters::network::http_connection_manager::v3:: std::string XdsBootstrapBuilder::Build() { std::vector fields; - fields.push_back(MakeXdsServersText(top_server_)); + fields.push_back(MakeXdsServersText(servers_)); if (!client_default_listener_resource_name_template_.empty()) { fields.push_back( absl::StrCat(" \"client_default_listener_resource_name_template\": \"", @@ -78,9 +78,8 @@ std::string XdsBootstrapBuilder::Build() { } std::string XdsBootstrapBuilder::MakeXdsServersText( - absl::string_view server_uri) { + absl::Span server_uris) { constexpr char kXdsServerTemplate[] = - " \"xds_servers\": [\n" " {\n" " \"server_uri\": \"\",\n" " \"channel_creds\": [\n" @@ -89,17 +88,21 @@ std::string XdsBootstrapBuilder::MakeXdsServersText( " }\n" " ],\n" " \"server_features\": []\n" - " }\n" - " ]"; + " }"; std::vector server_features; if (ignore_resource_deletion_) { server_features.push_back("\"ignore_resource_deletion\""); } - return absl::StrReplaceAll( - kXdsServerTemplate, - {{"", server_uri}, - {"", xds_channel_creds_type_}, - {"", absl::StrJoin(server_features, ", ")}}); + std::vector servers; + for (absl::string_view server_uri : server_uris) { + servers.emplace_back(absl::StrReplaceAll( + kXdsServerTemplate, + {{"", server_uri}, + {"", xds_channel_creds_type_}, + {"", absl::StrJoin(server_features, ", ")}})); + } + return absl::StrCat(" \"xds_servers\": [\n", + absl::StrJoin(servers, ",\n"), "\n ]"); } std::string XdsBootstrapBuilder::MakeNodeText() { @@ -148,7 +151,7 @@ std::string XdsBootstrapBuilder::MakeAuthorityText() { const std::string& name = p.first; const AuthorityInfo& authority_info = p.second; std::vector fields = { - MakeXdsServersText(authority_info.server)}; + MakeXdsServersText({authority_info.server})}; if (!authority_info.client_listener_resource_name_template.empty()) { fields.push_back(absl::StrCat( "\"client_listener_resource_name_template\": \"", diff --git a/test/cpp/end2end/xds/xds_utils.h b/test/cpp/end2end/xds/xds_utils.h index e12d36c2cfb..c3695125ae6 100644 --- a/test/cpp/end2end/xds/xds_utils.h +++ b/test/cpp/end2end/xds/xds_utils.h @@ -39,11 +39,8 @@ class XdsBootstrapBuilder { ignore_resource_deletion_ = true; return *this; } - // If ignore_if_set is true, sets the default server only if it has - // not already been set. - XdsBootstrapBuilder& SetDefaultServer(const std::string& server, - bool ignore_if_set = false) { - if (!ignore_if_set || top_server_.empty()) top_server_ = server; + XdsBootstrapBuilder& SetServers(absl::Span servers) { + servers_ = std::vector(servers.begin(), servers.end()); return *this; } XdsBootstrapBuilder& SetXdsChannelCredentials(const std::string& type) { @@ -87,13 +84,13 @@ class XdsBootstrapBuilder { std::string client_listener_resource_name_template; }; - std::string MakeXdsServersText(absl::string_view server_uri); + std::string MakeXdsServersText(absl::Span server_uris); std::string MakeNodeText(); std::string MakeCertificateProviderText(); std::string MakeAuthorityText(); bool ignore_resource_deletion_ = false; - std::string top_server_; + std::vector servers_; std::string xds_channel_creds_type_ = "fake"; std::string client_default_listener_resource_name_template_; std::map plugins_; From 6546fcd19640ead78749e135ab7514f2f5f93343 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Tue, 12 Mar 2024 13:42:52 -0700 Subject: [PATCH 34/52] [pick first] remove happy eyeballs experiment (#36092) Closes #36092 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/36092 from markdroth:pf_happy_eyeballs_cleanup 24d49849bb1b33b4a967c1acc39196f541674999 PiperOrigin-RevId: 615157870 --- bazel/experiments.bzl | 10 -- src/core/BUILD | 1 - src/core/lib/experiments/experiments.cc | 15 -- src/core/lib/experiments/experiments.h | 11 -- src/core/lib/experiments/experiments.yaml | 6 - src/core/lib/experiments/rollouts.yaml | 2 - .../load_balancing/pick_first/pick_first.cc | 160 +++--------------- .../lb_policy/pick_first_test.cc | 6 - 8 files changed, 19 insertions(+), 192 deletions(-) diff --git a/bazel/experiments.bzl b/bazel/experiments.bzl index 7e58d5cb29c..7edd8cd75ea 100644 --- a/bazel/experiments.bzl +++ b/bazel/experiments.bzl @@ -33,7 +33,6 @@ EXPERIMENT_ENABLES = { "multiping": "multiping", "peer_state_based_framing": "peer_state_based_framing", "pending_queue_cap": "pending_queue_cap", - "pick_first_happy_eyeballs": "pick_first_happy_eyeballs", "promise_based_client_call": "event_engine_client,event_engine_listener,promise_based_client_call", "promise_based_server_call": "promise_based_server_call", "chaotic_good": "chaotic_good,event_engine_client,event_engine_listener,promise_based_client_call,promise_based_server_call", @@ -103,7 +102,6 @@ EXPERIMENTS = { "event_engine_listener", ], "cpp_lb_end2end_test": [ - "pick_first_happy_eyeballs", "round_robin_delegate_to_pick_first", "wrr_delegate_to_pick_first", ], @@ -114,7 +112,6 @@ EXPERIMENTS = { "event_engine_listener", ], "lb_unit_test": [ - "pick_first_happy_eyeballs", "round_robin_delegate_to_pick_first", "wrr_delegate_to_pick_first", ], @@ -122,7 +119,6 @@ EXPERIMENTS = { "registered_method_lookup_in_transport", ], "xds_end2end_test": [ - "pick_first_happy_eyeballs", "round_robin_delegate_to_pick_first", "wrr_delegate_to_pick_first", ], @@ -165,7 +161,6 @@ EXPERIMENTS = { }, "on": { "cpp_lb_end2end_test": [ - "pick_first_happy_eyeballs", "round_robin_delegate_to_pick_first", "wrr_delegate_to_pick_first", ], @@ -173,7 +168,6 @@ EXPERIMENTS = { "absl_base64", ], "lb_unit_test": [ - "pick_first_happy_eyeballs", "round_robin_delegate_to_pick_first", "wrr_delegate_to_pick_first", ], @@ -181,7 +175,6 @@ EXPERIMENTS = { "registered_method_lookup_in_transport", ], "xds_end2end_test": [ - "pick_first_happy_eyeballs", "round_robin_delegate_to_pick_first", "wrr_delegate_to_pick_first", ], @@ -243,7 +236,6 @@ EXPERIMENTS = { "work_serializer_dispatch", ], "cpp_lb_end2end_test": [ - "pick_first_happy_eyeballs", "round_robin_delegate_to_pick_first", "wrr_delegate_to_pick_first", ], @@ -254,7 +246,6 @@ EXPERIMENTS = { "event_engine_listener", ], "lb_unit_test": [ - "pick_first_happy_eyeballs", "round_robin_delegate_to_pick_first", "work_serializer_dispatch", "wrr_delegate_to_pick_first", @@ -266,7 +257,6 @@ EXPERIMENTS = { "registered_method_lookup_in_transport", ], "xds_end2end_test": [ - "pick_first_happy_eyeballs", "round_robin_delegate_to_pick_first", "work_serializer_dispatch", "wrr_delegate_to_pick_first", diff --git a/src/core/BUILD b/src/core/BUILD index c591f75aee0..c67173ba22b 100644 --- a/src/core/BUILD +++ b/src/core/BUILD @@ -5555,7 +5555,6 @@ grpc_cc_library( deps = [ "channel_args", "connectivity_state", - "experiments", "health_check_client", "iomgr_fwd", "json", diff --git a/src/core/lib/experiments/experiments.cc b/src/core/lib/experiments/experiments.cc index 784735cd00c..495570303b8 100644 --- a/src/core/lib/experiments/experiments.cc +++ b/src/core/lib/experiments/experiments.cc @@ -80,9 +80,6 @@ const char* const description_pending_queue_cap = "grpc_server_request_call or grpc_server_request_registered_call (or their " "wrappers in the C++ API)."; const char* const additional_constraints_pending_queue_cap = "{}"; -const char* const description_pick_first_happy_eyeballs = - "Use Happy Eyeballs in pick_first."; -const char* const additional_constraints_pick_first_happy_eyeballs = "{}"; const char* const description_promise_based_client_call = "If set, use the new gRPC promise based call code when it's appropriate " "(ie when all filters in a stack are promise based)"; @@ -215,8 +212,6 @@ const ExperimentMetadata g_experiment_metadata[] = { additional_constraints_peer_state_based_framing, nullptr, 0, false, true}, {"pending_queue_cap", description_pending_queue_cap, additional_constraints_pending_queue_cap, nullptr, 0, true, true}, - {"pick_first_happy_eyeballs", description_pick_first_happy_eyeballs, - additional_constraints_pick_first_happy_eyeballs, nullptr, 0, true, true}, {"promise_based_client_call", description_promise_based_client_call, additional_constraints_promise_based_client_call, required_experiments_promise_based_client_call, 2, false, true}, @@ -334,9 +329,6 @@ const char* const description_pending_queue_cap = "grpc_server_request_call or grpc_server_request_registered_call (or their " "wrappers in the C++ API)."; const char* const additional_constraints_pending_queue_cap = "{}"; -const char* const description_pick_first_happy_eyeballs = - "Use Happy Eyeballs in pick_first."; -const char* const additional_constraints_pick_first_happy_eyeballs = "{}"; const char* const description_promise_based_client_call = "If set, use the new gRPC promise based call code when it's appropriate " "(ie when all filters in a stack are promise based)"; @@ -469,8 +461,6 @@ const ExperimentMetadata g_experiment_metadata[] = { additional_constraints_peer_state_based_framing, nullptr, 0, false, true}, {"pending_queue_cap", description_pending_queue_cap, additional_constraints_pending_queue_cap, nullptr, 0, true, true}, - {"pick_first_happy_eyeballs", description_pick_first_happy_eyeballs, - additional_constraints_pick_first_happy_eyeballs, nullptr, 0, true, true}, {"promise_based_client_call", description_promise_based_client_call, additional_constraints_promise_based_client_call, required_experiments_promise_based_client_call, 2, false, true}, @@ -588,9 +578,6 @@ const char* const description_pending_queue_cap = "grpc_server_request_call or grpc_server_request_registered_call (or their " "wrappers in the C++ API)."; const char* const additional_constraints_pending_queue_cap = "{}"; -const char* const description_pick_first_happy_eyeballs = - "Use Happy Eyeballs in pick_first."; -const char* const additional_constraints_pick_first_happy_eyeballs = "{}"; const char* const description_promise_based_client_call = "If set, use the new gRPC promise based call code when it's appropriate " "(ie when all filters in a stack are promise based)"; @@ -723,8 +710,6 @@ const ExperimentMetadata g_experiment_metadata[] = { additional_constraints_peer_state_based_framing, nullptr, 0, false, true}, {"pending_queue_cap", description_pending_queue_cap, additional_constraints_pending_queue_cap, nullptr, 0, true, true}, - {"pick_first_happy_eyeballs", description_pick_first_happy_eyeballs, - additional_constraints_pick_first_happy_eyeballs, nullptr, 0, true, true}, {"promise_based_client_call", description_promise_based_client_call, additional_constraints_promise_based_client_call, required_experiments_promise_based_client_call, 2, false, true}, diff --git a/src/core/lib/experiments/experiments.h b/src/core/lib/experiments/experiments.h index 1a51ba7799b..70027ee9d99 100644 --- a/src/core/lib/experiments/experiments.h +++ b/src/core/lib/experiments/experiments.h @@ -86,8 +86,6 @@ inline bool IsMultipingEnabled() { return false; } inline bool IsPeerStateBasedFramingEnabled() { return false; } #define GRPC_EXPERIMENT_IS_INCLUDED_PENDING_QUEUE_CAP inline bool IsPendingQueueCapEnabled() { return true; } -#define GRPC_EXPERIMENT_IS_INCLUDED_PICK_FIRST_HAPPY_EYEBALLS -inline bool IsPickFirstHappyEyeballsEnabled() { return true; } inline bool IsPromiseBasedClientCallEnabled() { return false; } inline bool IsPromiseBasedServerCallEnabled() { return false; } inline bool IsChaoticGoodEnabled() { return false; } @@ -144,8 +142,6 @@ inline bool IsMultipingEnabled() { return false; } inline bool IsPeerStateBasedFramingEnabled() { return false; } #define GRPC_EXPERIMENT_IS_INCLUDED_PENDING_QUEUE_CAP inline bool IsPendingQueueCapEnabled() { return true; } -#define GRPC_EXPERIMENT_IS_INCLUDED_PICK_FIRST_HAPPY_EYEBALLS -inline bool IsPickFirstHappyEyeballsEnabled() { return true; } inline bool IsPromiseBasedClientCallEnabled() { return false; } inline bool IsPromiseBasedServerCallEnabled() { return false; } inline bool IsChaoticGoodEnabled() { return false; } @@ -203,8 +199,6 @@ inline bool IsMultipingEnabled() { return false; } inline bool IsPeerStateBasedFramingEnabled() { return false; } #define GRPC_EXPERIMENT_IS_INCLUDED_PENDING_QUEUE_CAP inline bool IsPendingQueueCapEnabled() { return true; } -#define GRPC_EXPERIMENT_IS_INCLUDED_PICK_FIRST_HAPPY_EYEBALLS -inline bool IsPickFirstHappyEyeballsEnabled() { return true; } inline bool IsPromiseBasedClientCallEnabled() { return false; } inline bool IsPromiseBasedServerCallEnabled() { return false; } inline bool IsChaoticGoodEnabled() { return false; } @@ -250,7 +244,6 @@ enum ExperimentIds { kExperimentIdMultiping, kExperimentIdPeerStateBasedFraming, kExperimentIdPendingQueueCap, - kExperimentIdPickFirstHappyEyeballs, kExperimentIdPromiseBasedClientCall, kExperimentIdPromiseBasedServerCall, kExperimentIdChaoticGood, @@ -337,10 +330,6 @@ inline bool IsPeerStateBasedFramingEnabled() { inline bool IsPendingQueueCapEnabled() { return IsExperimentEnabled(kExperimentIdPendingQueueCap); } -#define GRPC_EXPERIMENT_IS_INCLUDED_PICK_FIRST_HAPPY_EYEBALLS -inline bool IsPickFirstHappyEyeballsEnabled() { - return IsExperimentEnabled(kExperimentIdPickFirstHappyEyeballs); -} #define GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_CLIENT_CALL inline bool IsPromiseBasedClientCallEnabled() { return IsExperimentEnabled(kExperimentIdPromiseBasedClientCall); diff --git a/src/core/lib/experiments/experiments.yaml b/src/core/lib/experiments/experiments.yaml index e67b4d4c92c..b86fde5cbdd 100644 --- a/src/core/lib/experiments/experiments.yaml +++ b/src/core/lib/experiments/experiments.yaml @@ -155,12 +155,6 @@ expiry: 2024/05/05 owner: ctiller@google.com test_tags: [] -- name: pick_first_happy_eyeballs - description: - Use Happy Eyeballs in pick_first. - expiry: 2024/03/15 - owner: roth@google.com - test_tags: ["lb_unit_test", "cpp_lb_end2end_test", "xds_end2end_test"] - name: promise_based_client_call description: If set, use the new gRPC promise based call code when it's appropriate diff --git a/src/core/lib/experiments/rollouts.yaml b/src/core/lib/experiments/rollouts.yaml index a971b0e4284..8bbd66af0ff 100644 --- a/src/core/lib/experiments/rollouts.yaml +++ b/src/core/lib/experiments/rollouts.yaml @@ -91,8 +91,6 @@ default: false - name: pending_queue_cap default: true -- name: pick_first_happy_eyeballs - default: true - name: promise_based_client_call default: ios: broken diff --git a/src/core/load_balancing/pick_first/pick_first.cc b/src/core/load_balancing/pick_first/pick_first.cc index 79c04248031..27c4de11db8 100644 --- a/src/core/load_balancing/pick_first/pick_first.cc +++ b/src/core/load_balancing/pick_first/pick_first.cc @@ -47,7 +47,6 @@ #include "src/core/lib/channel/metrics.h" #include "src/core/lib/config/core_configuration.h" #include "src/core/lib/debug/trace.h" -#include "src/core/lib/experiments/experiments.h" #include "src/core/lib/gpr/useful.h" #include "src/core/lib/gprpp/crash.h" #include "src/core/lib/gprpp/debug_location.h" @@ -198,10 +197,6 @@ class PickFirst : public LoadBalancingPolicy { // subchannel. void ProcessUnselectedReadyLocked(); - // Reacts to the current connectivity state while trying to connect. - // TODO(roth): Remove this when we remove the Happy Eyeballs experiment. - void ReactToConnectivityStateLocked(); - // Backpointer to owning subchannel list. Not owned. SubchannelList* subchannel_list_; const size_t index_; @@ -274,9 +269,6 @@ class PickFirst : public LoadBalancingPolicy { // finished processing. bool shutting_down_ = false; - // TODO(roth): Remove this when we remove the Happy Eyeballs experiment. - bool in_transient_failure_ = false; - size_t num_subchannels_seen_initial_notification_ = 0; // The index into subchannels_ to which we are currently attempting @@ -533,34 +525,30 @@ absl::Status PickFirst::UpdateLocked(UpdateArgs args) { for (const auto& endpoint : endpoints) { for (const auto& address : endpoint.addresses()) { flattened_endpoints.emplace_back(address, endpoint.args()); - if (IsPickFirstHappyEyeballsEnabled()) { - absl::string_view scheme = GetAddressFamily(address); - bool inserted = address_families.insert(scheme).second; - if (inserted) { - address_family_order.emplace_back(scheme, - flattened_endpoints.size() - 1); - } + absl::string_view scheme = GetAddressFamily(address); + bool inserted = address_families.insert(scheme).second; + if (inserted) { + address_family_order.emplace_back(scheme, + flattened_endpoints.size() - 1); } } } endpoints = std::move(flattened_endpoints); // Interleave addresses as per RFC-8305 section 4. - if (IsPickFirstHappyEyeballsEnabled()) { - EndpointAddressesList interleaved_endpoints; - interleaved_endpoints.reserve(endpoints.size()); - std::vector endpoints_moved(endpoints.size()); - size_t scheme_index = 0; - for (size_t i = 0; i < endpoints.size(); ++i) { - EndpointAddresses* endpoint; - do { - auto& iterator = address_family_order[scheme_index++ % - address_family_order.size()]; - endpoint = iterator.Next(endpoints, &endpoints_moved); - } while (endpoint == nullptr); - interleaved_endpoints.emplace_back(std::move(*endpoint)); - } - endpoints = std::move(interleaved_endpoints); + EndpointAddressesList interleaved_endpoints; + interleaved_endpoints.reserve(endpoints.size()); + std::vector endpoints_moved(endpoints.size()); + size_t scheme_index = 0; + for (size_t i = 0; i < endpoints.size(); ++i) { + EndpointAddresses* endpoint; + do { + auto& iterator = address_family_order[scheme_index++ % + address_family_order.size()]; + endpoint = iterator.Next(endpoints, &endpoints_moved); + } while (endpoint == nullptr); + interleaved_endpoints.emplace_back(std::move(*endpoint)); } + endpoints = std::move(interleaved_endpoints); args.addresses = std::make_shared(std::move(endpoints)); } @@ -738,9 +726,7 @@ void PickFirst::SubchannelList::SubchannelData::OnConnectivityStateChange( p->UnsetSelectedSubchannel(); p->subchannel_list_ = std::move(p->latest_pending_subchannel_list_); // Set our state to that of the pending subchannel list. - if (IsPickFirstHappyEyeballsEnabled() - ? p->subchannel_list_->IsHappyEyeballsPassComplete() - : p->subchannel_list_->in_transient_failure_) { + if (p->subchannel_list_->IsHappyEyeballsPassComplete()) { status = absl::UnavailableError(absl::StrCat( "selected subchannel failed; switching to pending update; " "last failure: ", @@ -772,9 +758,6 @@ void PickFirst::SubchannelList::SubchannelData::OnConnectivityStateChange( // select in place of the current one. // If the subchannel is READY, use it. if (new_state == GRPC_CHANNEL_READY) { - if (!IsPickFirstHappyEyeballsEnabled()) { - subchannel_list_->in_transient_failure_ = false; - } // We consider it a successful connection attempt only if the // previous state was CONNECTING. In particular, we don't want to // increment this counter if we got a new address list and found the @@ -807,10 +790,6 @@ void PickFirst::SubchannelList::SubchannelData::OnConnectivityStateChange( // see its initial notification. Start trying to connect, starting // with the first subchannel. if (!old_state.has_value()) { - if (!IsPickFirstHappyEyeballsEnabled()) { - subchannel_list_->subchannels_.front().ReactToConnectivityStateLocked(); - return; - } subchannel_list_->StartConnectingNextSubchannel(); return; } @@ -821,14 +800,6 @@ void PickFirst::SubchannelList::SubchannelData::OnConnectivityStateChange( kMetricConnectionAttemptsFailed, 1, {subchannel_list_->policy_->channel_control_helper()->GetTarget()}, {}); } - if (!IsPickFirstHappyEyeballsEnabled()) { - // Ignore any other updates for subchannels we're not currently trying to - // connect to. - if (index_ != subchannel_list_->attempting_index_) return; - // React to the connectivity state. - ReactToConnectivityStateLocked(); - return; - } // Otherwise, process connectivity state change. switch (*connectivity_state_) { case GRPC_CHANNEL_TRANSIENT_FAILURE: { @@ -898,99 +869,6 @@ void PickFirst::SubchannelList::SubchannelData::OnConnectivityStateChange( } } -void PickFirst::SubchannelList::SubchannelData:: - ReactToConnectivityStateLocked() { - PickFirst* p = subchannel_list_->policy_.get(); - // Otherwise, process connectivity state. - switch (connectivity_state_.value()) { - case GRPC_CHANNEL_READY: - // Already handled this case above, so this should not happen. - GPR_UNREACHABLE_CODE(break); - case GRPC_CHANNEL_TRANSIENT_FAILURE: { - // Find the next subchannel not in state TRANSIENT_FAILURE. - // We skip subchannels in state TRANSIENT_FAILURE to avoid a - // large recursion that could overflow the stack. - SubchannelData* found_subchannel = nullptr; - for (size_t next_index = index_ + 1; - next_index < subchannel_list_->size(); ++next_index) { - SubchannelData* sc = &subchannel_list_->subchannels_[next_index]; - GPR_ASSERT(sc->connectivity_state_.has_value()); - if (sc->connectivity_state_ != GRPC_CHANNEL_TRANSIENT_FAILURE) { - subchannel_list_->attempting_index_ = next_index; - found_subchannel = sc; - break; - } - } - // If we found another subchannel in the list not in state - // TRANSIENT_FAILURE, trigger the right behavior for that subchannel. - if (found_subchannel != nullptr) { - found_subchannel->ReactToConnectivityStateLocked(); - break; - } - // We didn't find another subchannel not in state TRANSIENT_FAILURE, - // so report TRANSIENT_FAILURE and wait for the first subchannel - // in the list to report IDLE before continuing. - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_pick_first_trace)) { - gpr_log(GPR_INFO, - "Pick First %p subchannel list %p failed to connect to " - "all subchannels", - p, subchannel_list_); - } - subchannel_list_->attempting_index_ = 0; - subchannel_list_->in_transient_failure_ = true; - // In case 2, swap to the new subchannel list. This means reporting - // TRANSIENT_FAILURE and dropping the existing (working) connection, - // but we can't ignore what the control plane has told us. - if (subchannel_list_ == p->latest_pending_subchannel_list_.get()) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_pick_first_trace)) { - gpr_log(GPR_INFO, - "Pick First %p promoting pending subchannel list %p to " - "replace %p", - p, p->latest_pending_subchannel_list_.get(), - p->subchannel_list_.get()); - } - p->UnsetSelectedSubchannel(); - p->subchannel_list_ = std::move(p->latest_pending_subchannel_list_); - } - // If this is the current subchannel list (either because we were - // in case 1 or because we were in case 2 and just promoted it to - // be the current list), re-resolve and report new state. - if (subchannel_list_ == p->subchannel_list_.get()) { - p->channel_control_helper()->RequestReresolution(); - absl::Status status = absl::UnavailableError(absl::StrCat( - (p->omit_status_message_prefix_ - ? "" - : "failed to connect to all addresses; last error: "), - connectivity_status_.ToString())); - p->UpdateState(GRPC_CHANNEL_TRANSIENT_FAILURE, status, - MakeRefCounted(status)); - } - // If the first subchannel is already IDLE, trigger the next connection - // attempt immediately. Otherwise, we'll wait for it to report - // its own connectivity state change. - auto& subchannel0 = subchannel_list_->subchannels_.front(); - if (subchannel0.connectivity_state_ == GRPC_CHANNEL_IDLE) { - subchannel0.subchannel_->RequestConnection(); - } - break; - } - case GRPC_CHANNEL_IDLE: - subchannel_->RequestConnection(); - break; - case GRPC_CHANNEL_CONNECTING: - // Only update connectivity state in case 1, and only if we're not - // already in TRANSIENT_FAILURE. - if (subchannel_list_ == p->subchannel_list_.get() && - p->state_ != GRPC_CHANNEL_TRANSIENT_FAILURE) { - p->UpdateState(GRPC_CHANNEL_CONNECTING, absl::Status(), - MakeRefCounted(nullptr)); - } - break; - case GRPC_CHANNEL_SHUTDOWN: - GPR_UNREACHABLE_CODE(break); - } -} - void PickFirst::SubchannelList::SubchannelData::RequestConnectionWithTimer() { GPR_ASSERT(connectivity_state_.has_value()); if (connectivity_state_ == GRPC_CHANNEL_IDLE) { diff --git a/test/core/client_channel/lb_policy/pick_first_test.cc b/test/core/client_channel/lb_policy/pick_first_test.cc index 05330ec9337..609f6f31a2f 100644 --- a/test/core/client_channel/lb_policy/pick_first_test.cc +++ b/test/core/client_channel/lb_policy/pick_first_test.cc @@ -38,7 +38,6 @@ #include #include "src/core/lib/channel/metrics.h" -#include "src/core/lib/experiments/experiments.h" #include "src/core/lib/gprpp/debug_location.h" #include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" @@ -513,7 +512,6 @@ TEST_F(PickFirstTest, ResolverUpdateBeforeLeavingIdle) { } TEST_F(PickFirstTest, HappyEyeballs) { - if (!IsPickFirstHappyEyeballsEnabled()) return; // Send an update containing three addresses. constexpr std::array kAddresses = { "ipv4:127.0.0.1:443", "ipv4:127.0.0.1:444", "ipv4:127.0.0.1:445"}; @@ -568,7 +566,6 @@ TEST_F(PickFirstTest, HappyEyeballs) { } TEST_F(PickFirstTest, HappyEyeballsCompletesWithoutSuccess) { - if (!IsPickFirstHappyEyeballsEnabled()) return; // Send an update containing three addresses. constexpr std::array kAddresses = { "ipv4:127.0.0.1:443", "ipv4:127.0.0.1:444", "ipv4:127.0.0.1:445"}; @@ -689,7 +686,6 @@ TEST_F(PickFirstTest, HappyEyeballsCompletesWithoutSuccess) { TEST_F(PickFirstTest, HappyEyeballsLastSubchannelFailsWhileAnotherIsStillPending) { - if (!IsPickFirstHappyEyeballsEnabled()) return; // Send an update containing three addresses. constexpr std::array kAddresses = { "ipv4:127.0.0.1:443", "ipv4:127.0.0.1:444"}; @@ -758,7 +754,6 @@ TEST_F(PickFirstTest, } TEST_F(PickFirstTest, HappyEyeballsAddressInterleaving) { - if (!IsPickFirstHappyEyeballsEnabled()) return; // Send an update containing four IPv4 addresses followed by two // IPv6 addresses. constexpr std::array kAddresses = { @@ -851,7 +846,6 @@ TEST_F(PickFirstTest, HappyEyeballsAddressInterleaving) { TEST_F(PickFirstTest, HappyEyeballsAddressInterleavingSecondFamilyHasMoreAddresses) { - if (!IsPickFirstHappyEyeballsEnabled()) return; // Send an update containing two IPv6 addresses followed by four IPv4 // addresses. constexpr std::array kAddresses = { From a19e01c4fd947ed5f4cc4a91412c4cbf029d2526 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Tue, 12 Mar 2024 15:56:15 -0700 Subject: [PATCH 35/52] [sanity] fix sanity on master (#36102) Broken by the merge commit for https://github.com/grpc/grpc/pull/36021. Closes #36102 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/36102 from markdroth:sanity 709aa26fe8b49632cb303c0c02cee424a97175e8 PiperOrigin-RevId: 615200273 --- src/core/ext/xds/xds_bootstrap_grpc.cc | 21 ++++++++++++--------- test/core/xds/xds_client_test.cc | 25 ++++++++++++++----------- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/src/core/ext/xds/xds_bootstrap_grpc.cc b/src/core/ext/xds/xds_bootstrap_grpc.cc index 553866d667b..41749e510cd 100644 --- a/src/core/ext/xds/xds_bootstrap_grpc.cc +++ b/src/core/ext/xds/xds_bootstrap_grpc.cc @@ -14,10 +14,10 @@ // limitations under the License. // +#include + #include "src/core/ext/xds/xds_bootstrap_grpc.h" -#include -#include #include #include @@ -26,13 +26,6 @@ #include #include -#include "src/core/lib/config/core_configuration.h" -#include "src/core/lib/gprpp/ref_counted_ptr.h" -#include "src/core/lib/json/json.h" -#include "src/core/lib/json/json_object_loader.h" -#include "src/core/lib/json/json_reader.h" -#include "src/core/lib/json/json_writer.h" -#include "src/core/lib/security/credentials/channel_creds_registry.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/match.h" @@ -42,6 +35,16 @@ #include "absl/strings/string_view.h" #include "absl/types/optional.h" +#include + +#include "src/core/lib/config/core_configuration.h" +#include "src/core/lib/gprpp/ref_counted_ptr.h" +#include "src/core/lib/json/json.h" +#include "src/core/lib/json/json_object_loader.h" +#include "src/core/lib/json/json_reader.h" +#include "src/core/lib/json/json_writer.h" +#include "src/core/lib/security/credentials/channel_creds_registry.h" + namespace grpc_core { // diff --git a/test/core/xds/xds_client_test.cc b/test/core/xds/xds_client_test.cc index c6365c03e73..741e2022362 100644 --- a/test/core/xds/xds_client_test.cc +++ b/test/core/xds/xds_client_test.cc @@ -20,12 +20,6 @@ #include "src/core/ext/xds/xds_client.h" -#include -#include -#include -#include -#include -#include #include #include @@ -35,8 +29,22 @@ #include #include +#include +#include + +#include "absl/strings/str_cat.h" +#include "absl/time/time.h" +#include "absl/types/optional.h" +#include "absl/types/variant.h" #include "gmock/gmock.h" #include "gtest/gtest.h" +#include "upb/reflection/def.h" + +#include +#include +#include +#include + #include "src/core/ext/xds/xds_bootstrap.h" #include "src/core/ext/xds/xds_resource_type_impl.h" #include "src/core/lib/event_engine/default_event_engine.h" @@ -52,11 +60,6 @@ #include "test/core/util/scoped_env_var.h" #include "test/core/util/test_config.h" #include "test/core/xds/xds_transport_fake.h" -#include "absl/strings/str_cat.h" -#include "absl/time/time.h" -#include "absl/types/optional.h" -#include "absl/types/variant.h" -#include "upb/reflection/def.h" // IWYU pragma: no_include // IWYU pragma: no_include From 390fef05901b602f317cf28cad29e8b9c9d7f501 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Tue, 12 Mar 2024 16:24:17 -0700 Subject: [PATCH 36/52] [RR and WRR] clean up dualstack experiments (#35135) Closes #35135 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/35135 from markdroth:dualstack_rr_wrr_cleanup 438b29df5ca83c0020f05cb437369720622ed752 PiperOrigin-RevId: 615208297 --- Package.swift | 1 - bazel/experiments.bzl | 34 - build_autogenerated.yaml | 2 - gRPC-C++.podspec | 2 - gRPC-Core.podspec | 2 - grpc.gemspec | 1 - package.xml | 1 - src/core/BUILD | 36 - src/core/lib/experiments/experiments.cc | 45 - src/core/lib/experiments/experiments.h | 22 - src/core/lib/experiments/experiments.yaml | 14 - src/core/lib/experiments/rollouts.yaml | 4 - .../load_balancing/round_robin/round_robin.cc | 457 +------ src/core/load_balancing/subchannel_list.h | 455 ------- .../weighted_round_robin.cc | 1051 ++--------------- .../lb_policy/outlier_detection_test.cc | 4 - .../lb_policy/round_robin_test.cc | 2 - .../lb_policy/weighted_round_robin_test.cc | 3 - .../lb_policy/xds_override_host_test.cc | 2 - test/cpp/end2end/client_lb_end2end_test.cc | 28 +- .../end2end/xds/xds_cluster_end2end_test.cc | 2 - .../xds/xds_override_host_end2end_test.cc | 2 - test/cpp/end2end/xds/xds_wrr_end2end_test.cc | 2 - tools/doxygen/Doxyfile.c++.internal | 1 - tools/doxygen/Doxyfile.core.internal | 1 - 25 files changed, 103 insertions(+), 2071 deletions(-) delete mode 100644 src/core/load_balancing/subchannel_list.h diff --git a/Package.swift b/Package.swift index 5e15d8e55af..0465d912aa5 100644 --- a/Package.swift +++ b/Package.swift @@ -1896,7 +1896,6 @@ let package = Package( "src/core/load_balancing/rls/rls.h", "src/core/load_balancing/round_robin/round_robin.cc", "src/core/load_balancing/subchannel_interface.h", - "src/core/load_balancing/subchannel_list.h", "src/core/load_balancing/weighted_round_robin/static_stride_scheduler.cc", "src/core/load_balancing/weighted_round_robin/static_stride_scheduler.h", "src/core/load_balancing/weighted_round_robin/weighted_round_robin.cc", diff --git a/bazel/experiments.bzl b/bazel/experiments.bzl index 7edd8cd75ea..79de231736b 100644 --- a/bazel/experiments.bzl +++ b/bazel/experiments.bzl @@ -38,7 +38,6 @@ EXPERIMENT_ENABLES = { "chaotic_good": "chaotic_good,event_engine_client,event_engine_listener,promise_based_client_call,promise_based_server_call", "registered_method_lookup_in_transport": "registered_method_lookup_in_transport", "promise_based_inproc_transport": "event_engine_client,event_engine_listener,promise_based_client_call,promise_based_inproc_transport,promise_based_server_call,registered_method_lookup_in_transport", - "round_robin_delegate_to_pick_first": "round_robin_delegate_to_pick_first", "rstpit": "rstpit", "schedule_cancellation_over_write": "schedule_cancellation_over_write", "server_privacy": "server_privacy", @@ -52,7 +51,6 @@ EXPERIMENT_ENABLES = { "v3_server_auth_filter": "v3_server_auth_filter", "work_serializer_clears_time_cache": "work_serializer_clears_time_cache", "work_serializer_dispatch": "event_engine_client,work_serializer_dispatch", - "wrr_delegate_to_pick_first": "wrr_delegate_to_pick_first", } EXPERIMENT_POLLERS = [ @@ -101,27 +99,15 @@ EXPERIMENTS = { "core_end2end_test": [ "event_engine_listener", ], - "cpp_lb_end2end_test": [ - "round_robin_delegate_to_pick_first", - "wrr_delegate_to_pick_first", - ], "credential_token_tests": [ "absl_base64", ], "event_engine_listener_test": [ "event_engine_listener", ], - "lb_unit_test": [ - "round_robin_delegate_to_pick_first", - "wrr_delegate_to_pick_first", - ], "surface_registered_method_lookup": [ "registered_method_lookup_in_transport", ], - "xds_end2end_test": [ - "round_robin_delegate_to_pick_first", - "wrr_delegate_to_pick_first", - ], }, }, "ios": { @@ -160,24 +146,12 @@ EXPERIMENTS = { ], }, "on": { - "cpp_lb_end2end_test": [ - "round_robin_delegate_to_pick_first", - "wrr_delegate_to_pick_first", - ], "credential_token_tests": [ "absl_base64", ], - "lb_unit_test": [ - "round_robin_delegate_to_pick_first", - "wrr_delegate_to_pick_first", - ], "surface_registered_method_lookup": [ "registered_method_lookup_in_transport", ], - "xds_end2end_test": [ - "round_robin_delegate_to_pick_first", - "wrr_delegate_to_pick_first", - ], }, }, "posix": { @@ -235,10 +209,6 @@ EXPERIMENTS = { "cpp_end2end_test": [ "work_serializer_dispatch", ], - "cpp_lb_end2end_test": [ - "round_robin_delegate_to_pick_first", - "wrr_delegate_to_pick_first", - ], "credential_token_tests": [ "absl_base64", ], @@ -246,9 +216,7 @@ EXPERIMENTS = { "event_engine_listener", ], "lb_unit_test": [ - "round_robin_delegate_to_pick_first", "work_serializer_dispatch", - "wrr_delegate_to_pick_first", ], "resolver_component_tests_runner_invoker": [ "event_engine_dns", @@ -257,9 +225,7 @@ EXPERIMENTS = { "registered_method_lookup_in_transport", ], "xds_end2end_test": [ - "round_robin_delegate_to_pick_first", "work_serializer_dispatch", - "wrr_delegate_to_pick_first", ], }, }, diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index a3f46868f21..91e82652419 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -1183,7 +1183,6 @@ libs: - src/core/load_balancing/ring_hash/ring_hash.h - src/core/load_balancing/rls/rls.h - src/core/load_balancing/subchannel_interface.h - - src/core/load_balancing/subchannel_list.h - src/core/load_balancing/weighted_round_robin/static_stride_scheduler.h - src/core/load_balancing/weighted_target/weighted_target.h - src/core/load_balancing/xds/xds_channel_args.h @@ -2655,7 +2654,6 @@ libs: - src/core/load_balancing/pick_first/pick_first.h - src/core/load_balancing/rls/rls.h - src/core/load_balancing/subchannel_interface.h - - src/core/load_balancing/subchannel_list.h - src/core/load_balancing/weighted_round_robin/static_stride_scheduler.h - src/core/load_balancing/weighted_target/weighted_target.h - src/core/resolver/dns/c_ares/dns_resolver_ares.h diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index 898b33d3655..f459d1b7b85 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -1289,7 +1289,6 @@ Pod::Spec.new do |s| 'src/core/load_balancing/ring_hash/ring_hash.h', 'src/core/load_balancing/rls/rls.h', 'src/core/load_balancing/subchannel_interface.h', - 'src/core/load_balancing/subchannel_list.h', 'src/core/load_balancing/weighted_round_robin/static_stride_scheduler.h', 'src/core/load_balancing/weighted_target/weighted_target.h', 'src/core/load_balancing/xds/xds_channel_args.h', @@ -2554,7 +2553,6 @@ Pod::Spec.new do |s| 'src/core/load_balancing/ring_hash/ring_hash.h', 'src/core/load_balancing/rls/rls.h', 'src/core/load_balancing/subchannel_interface.h', - 'src/core/load_balancing/subchannel_list.h', 'src/core/load_balancing/weighted_round_robin/static_stride_scheduler.h', 'src/core/load_balancing/weighted_target/weighted_target.h', 'src/core/load_balancing/xds/xds_channel_args.h', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 8326e6b78bd..dd30ea8829c 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -2006,7 +2006,6 @@ Pod::Spec.new do |s| 'src/core/load_balancing/rls/rls.h', 'src/core/load_balancing/round_robin/round_robin.cc', 'src/core/load_balancing/subchannel_interface.h', - 'src/core/load_balancing/subchannel_list.h', 'src/core/load_balancing/weighted_round_robin/static_stride_scheduler.cc', 'src/core/load_balancing/weighted_round_robin/static_stride_scheduler.h', 'src/core/load_balancing/weighted_round_robin/weighted_round_robin.cc', @@ -3336,7 +3335,6 @@ Pod::Spec.new do |s| 'src/core/load_balancing/ring_hash/ring_hash.h', 'src/core/load_balancing/rls/rls.h', 'src/core/load_balancing/subchannel_interface.h', - 'src/core/load_balancing/subchannel_list.h', 'src/core/load_balancing/weighted_round_robin/static_stride_scheduler.h', 'src/core/load_balancing/weighted_target/weighted_target.h', 'src/core/load_balancing/xds/xds_channel_args.h', diff --git a/grpc.gemspec b/grpc.gemspec index 8b33f403ed2..64b5cf62e4b 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -1898,7 +1898,6 @@ Gem::Specification.new do |s| s.files += %w( src/core/load_balancing/rls/rls.h ) s.files += %w( src/core/load_balancing/round_robin/round_robin.cc ) s.files += %w( src/core/load_balancing/subchannel_interface.h ) - s.files += %w( src/core/load_balancing/subchannel_list.h ) s.files += %w( src/core/load_balancing/weighted_round_robin/static_stride_scheduler.cc ) s.files += %w( src/core/load_balancing/weighted_round_robin/static_stride_scheduler.h ) s.files += %w( src/core/load_balancing/weighted_round_robin/weighted_round_robin.cc ) diff --git a/package.xml b/package.xml index cfe611d7459..84621351c14 100644 --- a/package.xml +++ b/package.xml @@ -1880,7 +1880,6 @@ - diff --git a/src/core/BUILD b/src/core/BUILD index c67173ba22b..857484df4f1 100644 --- a/src/core/BUILD +++ b/src/core/BUILD @@ -5470,35 +5470,6 @@ grpc_cc_library( ], ) -grpc_cc_library( - name = "grpc_lb_subchannel_list", - hdrs = [ - "load_balancing/subchannel_list.h", - ], - external_deps = [ - "absl/status", - "absl/types:optional", - ], - language = "c++", - deps = [ - "channel_args", - "connectivity_state", - "dual_ref_counted", - "gpr_manual_constructor", - "health_check_client", - "iomgr_fwd", - "lb_policy", - "subchannel_interface", - "//:debug_location", - "//:endpoint_addresses", - "//:gpr", - "//:grpc_base", - "//:ref_counted_ptr", - "//:server_address", - "//:work_serializer", - ], -) - grpc_cc_library( name = "lb_endpoint_list", srcs = [ @@ -5721,13 +5692,10 @@ grpc_cc_library( deps = [ "channel_args", "connectivity_state", - "experiments", - "grpc_lb_subchannel_list", "json", "lb_endpoint_list", "lb_policy", "lb_policy_factory", - "subchannel_interface", "//:config", "//:debug_location", "//:endpoint_addresses", @@ -5736,7 +5704,6 @@ grpc_cc_library( "//:grpc_trace", "//:orphanable", "//:ref_counted_ptr", - "//:server_address", "//:work_serializer", ], ) @@ -5780,7 +5747,6 @@ grpc_cc_library( "experiments", "grpc_backend_metric_data", "grpc_lb_policy_weighted_target", - "grpc_lb_subchannel_list", "json", "json_args", "json_object_loader", @@ -5806,8 +5772,6 @@ grpc_cc_library( "//:oob_backend_metric", "//:orphanable", "//:ref_counted_ptr", - "//:server_address", - "//:sockaddr_utils", "//:stats", "//:work_serializer", ], diff --git a/src/core/lib/experiments/experiments.cc b/src/core/lib/experiments/experiments.cc index 495570303b8..41307cf8843 100644 --- a/src/core/lib/experiments/experiments.cc +++ b/src/core/lib/experiments/experiments.cc @@ -110,11 +110,6 @@ const uint8_t required_experiments_promise_based_inproc_transport[] = { static_cast(grpc_core::kExperimentIdPromiseBasedServerCall), static_cast( grpc_core::kExperimentIdRegisteredMethodLookupInTransport)}; -const char* const description_round_robin_delegate_to_pick_first = - "Change round_robin code to delegate to pick_first as per dualstack " - "backend design."; -const char* const additional_constraints_round_robin_delegate_to_pick_first = - "{}"; const char* const description_rstpit = "On RST_STREAM on a server, reduce MAX_CONCURRENT_STREAMS for a short " "duration"; @@ -164,10 +159,6 @@ const char* const description_work_serializer_dispatch = const char* const additional_constraints_work_serializer_dispatch = "{}"; const uint8_t required_experiments_work_serializer_dispatch[] = { static_cast(grpc_core::kExperimentIdEventEngineClient)}; -const char* const description_wrr_delegate_to_pick_first = - "Change WRR code to delegate to pick_first as per dualstack backend " - "design."; -const char* const additional_constraints_wrr_delegate_to_pick_first = "{}"; #ifdef NDEBUG const bool kDefaultForDebugOnly = false; #else @@ -228,10 +219,6 @@ const ExperimentMetadata g_experiment_metadata[] = { description_promise_based_inproc_transport, additional_constraints_promise_based_inproc_transport, required_experiments_promise_based_inproc_transport, 3, false, false}, - {"round_robin_delegate_to_pick_first", - description_round_robin_delegate_to_pick_first, - additional_constraints_round_robin_delegate_to_pick_first, nullptr, 0, - true, true}, {"rstpit", description_rstpit, additional_constraints_rstpit, nullptr, 0, false, true}, {"schedule_cancellation_over_write", @@ -265,8 +252,6 @@ const ExperimentMetadata g_experiment_metadata[] = { {"work_serializer_dispatch", description_work_serializer_dispatch, additional_constraints_work_serializer_dispatch, required_experiments_work_serializer_dispatch, 1, false, true}, - {"wrr_delegate_to_pick_first", description_wrr_delegate_to_pick_first, - additional_constraints_wrr_delegate_to_pick_first, nullptr, 0, true, true}, }; } // namespace grpc_core @@ -359,11 +344,6 @@ const uint8_t required_experiments_promise_based_inproc_transport[] = { static_cast(grpc_core::kExperimentIdPromiseBasedServerCall), static_cast( grpc_core::kExperimentIdRegisteredMethodLookupInTransport)}; -const char* const description_round_robin_delegate_to_pick_first = - "Change round_robin code to delegate to pick_first as per dualstack " - "backend design."; -const char* const additional_constraints_round_robin_delegate_to_pick_first = - "{}"; const char* const description_rstpit = "On RST_STREAM on a server, reduce MAX_CONCURRENT_STREAMS for a short " "duration"; @@ -413,10 +393,6 @@ const char* const description_work_serializer_dispatch = const char* const additional_constraints_work_serializer_dispatch = "{}"; const uint8_t required_experiments_work_serializer_dispatch[] = { static_cast(grpc_core::kExperimentIdEventEngineClient)}; -const char* const description_wrr_delegate_to_pick_first = - "Change WRR code to delegate to pick_first as per dualstack backend " - "design."; -const char* const additional_constraints_wrr_delegate_to_pick_first = "{}"; #ifdef NDEBUG const bool kDefaultForDebugOnly = false; #else @@ -477,10 +453,6 @@ const ExperimentMetadata g_experiment_metadata[] = { description_promise_based_inproc_transport, additional_constraints_promise_based_inproc_transport, required_experiments_promise_based_inproc_transport, 3, false, false}, - {"round_robin_delegate_to_pick_first", - description_round_robin_delegate_to_pick_first, - additional_constraints_round_robin_delegate_to_pick_first, nullptr, 0, - true, true}, {"rstpit", description_rstpit, additional_constraints_rstpit, nullptr, 0, false, true}, {"schedule_cancellation_over_write", @@ -514,8 +486,6 @@ const ExperimentMetadata g_experiment_metadata[] = { {"work_serializer_dispatch", description_work_serializer_dispatch, additional_constraints_work_serializer_dispatch, required_experiments_work_serializer_dispatch, 1, false, true}, - {"wrr_delegate_to_pick_first", description_wrr_delegate_to_pick_first, - additional_constraints_wrr_delegate_to_pick_first, nullptr, 0, true, true}, }; } // namespace grpc_core @@ -608,11 +578,6 @@ const uint8_t required_experiments_promise_based_inproc_transport[] = { static_cast(grpc_core::kExperimentIdPromiseBasedServerCall), static_cast( grpc_core::kExperimentIdRegisteredMethodLookupInTransport)}; -const char* const description_round_robin_delegate_to_pick_first = - "Change round_robin code to delegate to pick_first as per dualstack " - "backend design."; -const char* const additional_constraints_round_robin_delegate_to_pick_first = - "{}"; const char* const description_rstpit = "On RST_STREAM on a server, reduce MAX_CONCURRENT_STREAMS for a short " "duration"; @@ -662,10 +627,6 @@ const char* const description_work_serializer_dispatch = const char* const additional_constraints_work_serializer_dispatch = "{}"; const uint8_t required_experiments_work_serializer_dispatch[] = { static_cast(grpc_core::kExperimentIdEventEngineClient)}; -const char* const description_wrr_delegate_to_pick_first = - "Change WRR code to delegate to pick_first as per dualstack backend " - "design."; -const char* const additional_constraints_wrr_delegate_to_pick_first = "{}"; #ifdef NDEBUG const bool kDefaultForDebugOnly = false; #else @@ -726,10 +687,6 @@ const ExperimentMetadata g_experiment_metadata[] = { description_promise_based_inproc_transport, additional_constraints_promise_based_inproc_transport, required_experiments_promise_based_inproc_transport, 3, false, false}, - {"round_robin_delegate_to_pick_first", - description_round_robin_delegate_to_pick_first, - additional_constraints_round_robin_delegate_to_pick_first, nullptr, 0, - true, true}, {"rstpit", description_rstpit, additional_constraints_rstpit, nullptr, 0, false, true}, {"schedule_cancellation_over_write", @@ -763,8 +720,6 @@ const ExperimentMetadata g_experiment_metadata[] = { {"work_serializer_dispatch", description_work_serializer_dispatch, additional_constraints_work_serializer_dispatch, required_experiments_work_serializer_dispatch, 1, true, true}, - {"wrr_delegate_to_pick_first", description_wrr_delegate_to_pick_first, - additional_constraints_wrr_delegate_to_pick_first, nullptr, 0, true, true}, }; } // namespace grpc_core diff --git a/src/core/lib/experiments/experiments.h b/src/core/lib/experiments/experiments.h index 70027ee9d99..90e66aee6f5 100644 --- a/src/core/lib/experiments/experiments.h +++ b/src/core/lib/experiments/experiments.h @@ -92,8 +92,6 @@ inline bool IsChaoticGoodEnabled() { return false; } #define GRPC_EXPERIMENT_IS_INCLUDED_REGISTERED_METHOD_LOOKUP_IN_TRANSPORT inline bool IsRegisteredMethodLookupInTransportEnabled() { return true; } inline bool IsPromiseBasedInprocTransportEnabled() { return false; } -#define GRPC_EXPERIMENT_IS_INCLUDED_ROUND_ROBIN_DELEGATE_TO_PICK_FIRST -inline bool IsRoundRobinDelegateToPickFirstEnabled() { return true; } inline bool IsRstpitEnabled() { return false; } inline bool IsScheduleCancellationOverWriteEnabled() { return false; } inline bool IsServerPrivacyEnabled() { return false; } @@ -108,8 +106,6 @@ inline bool IsV3ServerAuthFilterEnabled() { return false; } #define GRPC_EXPERIMENT_IS_INCLUDED_WORK_SERIALIZER_CLEARS_TIME_CACHE inline bool IsWorkSerializerClearsTimeCacheEnabled() { return true; } inline bool IsWorkSerializerDispatchEnabled() { return false; } -#define GRPC_EXPERIMENT_IS_INCLUDED_WRR_DELEGATE_TO_PICK_FIRST -inline bool IsWrrDelegateToPickFirstEnabled() { return true; } #elif defined(GPR_WINDOWS) #define GRPC_EXPERIMENT_IS_INCLUDED_ABSL_BASE64 @@ -148,8 +144,6 @@ inline bool IsChaoticGoodEnabled() { return false; } #define GRPC_EXPERIMENT_IS_INCLUDED_REGISTERED_METHOD_LOOKUP_IN_TRANSPORT inline bool IsRegisteredMethodLookupInTransportEnabled() { return true; } inline bool IsPromiseBasedInprocTransportEnabled() { return false; } -#define GRPC_EXPERIMENT_IS_INCLUDED_ROUND_ROBIN_DELEGATE_TO_PICK_FIRST -inline bool IsRoundRobinDelegateToPickFirstEnabled() { return true; } inline bool IsRstpitEnabled() { return false; } inline bool IsScheduleCancellationOverWriteEnabled() { return false; } inline bool IsServerPrivacyEnabled() { return false; } @@ -164,8 +158,6 @@ inline bool IsV3ServerAuthFilterEnabled() { return false; } #define GRPC_EXPERIMENT_IS_INCLUDED_WORK_SERIALIZER_CLEARS_TIME_CACHE inline bool IsWorkSerializerClearsTimeCacheEnabled() { return true; } inline bool IsWorkSerializerDispatchEnabled() { return false; } -#define GRPC_EXPERIMENT_IS_INCLUDED_WRR_DELEGATE_TO_PICK_FIRST -inline bool IsWrrDelegateToPickFirstEnabled() { return true; } #else #define GRPC_EXPERIMENT_IS_INCLUDED_ABSL_BASE64 @@ -205,8 +197,6 @@ inline bool IsChaoticGoodEnabled() { return false; } #define GRPC_EXPERIMENT_IS_INCLUDED_REGISTERED_METHOD_LOOKUP_IN_TRANSPORT inline bool IsRegisteredMethodLookupInTransportEnabled() { return true; } inline bool IsPromiseBasedInprocTransportEnabled() { return false; } -#define GRPC_EXPERIMENT_IS_INCLUDED_ROUND_ROBIN_DELEGATE_TO_PICK_FIRST -inline bool IsRoundRobinDelegateToPickFirstEnabled() { return true; } inline bool IsRstpitEnabled() { return false; } inline bool IsScheduleCancellationOverWriteEnabled() { return false; } inline bool IsServerPrivacyEnabled() { return false; } @@ -222,8 +212,6 @@ inline bool IsV3ServerAuthFilterEnabled() { return false; } inline bool IsWorkSerializerClearsTimeCacheEnabled() { return true; } #define GRPC_EXPERIMENT_IS_INCLUDED_WORK_SERIALIZER_DISPATCH inline bool IsWorkSerializerDispatchEnabled() { return true; } -#define GRPC_EXPERIMENT_IS_INCLUDED_WRR_DELEGATE_TO_PICK_FIRST -inline bool IsWrrDelegateToPickFirstEnabled() { return true; } #endif #else @@ -249,7 +237,6 @@ enum ExperimentIds { kExperimentIdChaoticGood, kExperimentIdRegisteredMethodLookupInTransport, kExperimentIdPromiseBasedInprocTransport, - kExperimentIdRoundRobinDelegateToPickFirst, kExperimentIdRstpit, kExperimentIdScheduleCancellationOverWrite, kExperimentIdServerPrivacy, @@ -263,7 +250,6 @@ enum ExperimentIds { kExperimentIdV3ServerAuthFilter, kExperimentIdWorkSerializerClearsTimeCache, kExperimentIdWorkSerializerDispatch, - kExperimentIdWrrDelegateToPickFirst, kNumExperiments }; #define GRPC_EXPERIMENT_IS_INCLUDED_ABSL_BASE64 @@ -350,10 +336,6 @@ inline bool IsRegisteredMethodLookupInTransportEnabled() { inline bool IsPromiseBasedInprocTransportEnabled() { return IsExperimentEnabled(kExperimentIdPromiseBasedInprocTransport); } -#define GRPC_EXPERIMENT_IS_INCLUDED_ROUND_ROBIN_DELEGATE_TO_PICK_FIRST -inline bool IsRoundRobinDelegateToPickFirstEnabled() { - return IsExperimentEnabled(kExperimentIdRoundRobinDelegateToPickFirst); -} #define GRPC_EXPERIMENT_IS_INCLUDED_RSTPIT inline bool IsRstpitEnabled() { return IsExperimentEnabled(kExperimentIdRstpit); @@ -406,10 +388,6 @@ inline bool IsWorkSerializerClearsTimeCacheEnabled() { inline bool IsWorkSerializerDispatchEnabled() { return IsExperimentEnabled(kExperimentIdWorkSerializerDispatch); } -#define GRPC_EXPERIMENT_IS_INCLUDED_WRR_DELEGATE_TO_PICK_FIRST -inline bool IsWrrDelegateToPickFirstEnabled() { - return IsExperimentEnabled(kExperimentIdWrrDelegateToPickFirst); -} extern const ExperimentMetadata g_experiment_metadata[kNumExperiments]; diff --git a/src/core/lib/experiments/experiments.yaml b/src/core/lib/experiments/experiments.yaml index b86fde5cbdd..f1c1f4a11fc 100644 --- a/src/core/lib/experiments/experiments.yaml +++ b/src/core/lib/experiments/experiments.yaml @@ -184,13 +184,6 @@ expiry: 2024/03/31 owner: yashkt@google.com test_tags: ["surface_registered_method_lookup"] -- name: round_robin_delegate_to_pick_first - description: - Change round_robin code to delegate to pick_first as per dualstack - backend design. - expiry: 2024/03/15 - owner: roth@google.com - test_tags: ["lb_unit_test", "cpp_lb_end2end_test", "xds_end2end_test"] - name: rstpit description: On RST_STREAM on a server, reduce MAX_CONCURRENT_STREAMS for a short duration @@ -271,10 +264,3 @@ expiry: 2024/03/31 owner: ysseung@google.com test_tags: ["core_end2end_test", "cpp_end2end_test", "xds_end2end_test", "lb_unit_test"] -- name: wrr_delegate_to_pick_first - description: - Change WRR code to delegate to pick_first as per dualstack - backend design. - expiry: 2024/03/15 - owner: roth@google.com - test_tags: ["lb_unit_test", "cpp_lb_end2end_test", "xds_end2end_test"] diff --git a/src/core/lib/experiments/rollouts.yaml b/src/core/lib/experiments/rollouts.yaml index 8bbd66af0ff..b6cbcd468a2 100644 --- a/src/core/lib/experiments/rollouts.yaml +++ b/src/core/lib/experiments/rollouts.yaml @@ -100,8 +100,6 @@ default: false - name: registered_method_lookup_in_transport default: true -- name: round_robin_delegate_to_pick_first - default: true - name: rstpit default: false - name: schedule_cancellation_over_write @@ -126,5 +124,3 @@ posix: true # TODO(ysseung): Test flakes not fully resolved. windows: broken -- name: wrr_delegate_to_pick_first - default: true diff --git a/src/core/load_balancing/round_robin/round_robin.cc b/src/core/load_balancing/round_robin/round_robin.cc index 66065aa06bf..1bfa55ccbaa 100644 --- a/src/core/load_balancing/round_robin/round_robin.cc +++ b/src/core/load_balancing/round_robin/round_robin.cc @@ -37,23 +37,19 @@ #include #include -#include "src/core/load_balancing/endpoint_list.h" -#include "src/core/load_balancing/subchannel_list.h" #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/config/core_configuration.h" #include "src/core/lib/debug/trace.h" -#include "src/core/lib/experiments/experiments.h" #include "src/core/lib/gprpp/debug_location.h" #include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/gprpp/work_serializer.h" #include "src/core/lib/json/json.h" #include "src/core/lib/transport/connectivity_state.h" +#include "src/core/load_balancing/endpoint_list.h" #include "src/core/load_balancing/lb_policy.h" #include "src/core/load_balancing/lb_policy_factory.h" -#include "src/core/load_balancing/subchannel_interface.h" #include "src/core/resolver/endpoint_addresses.h" -#include "src/core/resolver/server_address.h" namespace grpc_core { @@ -61,456 +57,8 @@ TraceFlag grpc_lb_round_robin_trace(false, "round_robin"); namespace { -// -// legacy round_robin LB policy (before dualstack support) -// - constexpr absl::string_view kRoundRobin = "round_robin"; -class OldRoundRobin : public LoadBalancingPolicy { - public: - explicit OldRoundRobin(Args args); - - absl::string_view name() const override { return kRoundRobin; } - - absl::Status UpdateLocked(UpdateArgs args) override; - void ResetBackoffLocked() override; - - private: - ~OldRoundRobin() override; - - // Forward declaration. - class RoundRobinSubchannelList; - - // Data for a particular subchannel in a subchannel list. - // This subclass adds the following functionality: - // - Tracks the previous connectivity state of the subchannel, so that - // we know how many subchannels are in each state. - class RoundRobinSubchannelData - : public SubchannelData { - public: - RoundRobinSubchannelData( - SubchannelList* - subchannel_list, - const ServerAddress& address, - RefCountedPtr subchannel) - : SubchannelData(subchannel_list, address, std::move(subchannel)) {} - - absl::optional connectivity_state() const { - return logical_connectivity_state_; - } - - private: - // Performs connectivity state updates that need to be done only - // after we have started watching. - void ProcessConnectivityChangeLocked( - absl::optional old_state, - grpc_connectivity_state new_state) override; - - // Updates the logical connectivity state. - void UpdateLogicalConnectivityStateLocked( - grpc_connectivity_state connectivity_state); - - // The logical connectivity state of the subchannel. - // Note that the logical connectivity state may differ from the - // actual reported state in some cases (e.g., after we see - // TRANSIENT_FAILURE, we ignore any subsequent state changes until - // we see READY). - absl::optional logical_connectivity_state_; - }; - - // A list of subchannels. - class RoundRobinSubchannelList - : public SubchannelList { - public: - RoundRobinSubchannelList(OldRoundRobin* policy, - EndpointAddressesIterator* addresses, - const ChannelArgs& args) - : SubchannelList(policy, - (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace) - ? "RoundRobinSubchannelList" - : nullptr), - addresses, policy->channel_control_helper(), args) { - // Need to maintain a ref to the LB policy as long as we maintain - // any references to subchannels, since the subchannels' - // pollset_sets will include the LB policy's pollset_set. - policy->Ref(DEBUG_LOCATION, "subchannel_list").release(); - } - - ~RoundRobinSubchannelList() override { - OldRoundRobin* p = static_cast(policy()); - p->Unref(DEBUG_LOCATION, "subchannel_list"); - } - - // Updates the counters of subchannels in each state when a - // subchannel transitions from old_state to new_state. - void UpdateStateCountersLocked( - absl::optional old_state, - grpc_connectivity_state new_state); - - // Ensures that the right subchannel list is used and then updates - // the RR policy's connectivity state based on the subchannel list's - // state counters. - void MaybeUpdateRoundRobinConnectivityStateLocked( - absl::Status status_for_tf); - - private: - std::shared_ptr work_serializer() const override { - return static_cast(policy())->work_serializer(); - } - - std::string CountersString() const { - return absl::StrCat("num_subchannels=", num_subchannels(), - " num_ready=", num_ready_, - " num_connecting=", num_connecting_, - " num_transient_failure=", num_transient_failure_); - } - - size_t num_ready_ = 0; - size_t num_connecting_ = 0; - size_t num_transient_failure_ = 0; - - absl::Status last_failure_; - }; - - class Picker : public SubchannelPicker { - public: - Picker(OldRoundRobin* parent, RoundRobinSubchannelList* subchannel_list); - - PickResult Pick(PickArgs args) override; - - private: - // Using pointer value only, no ref held -- do not dereference! - OldRoundRobin* parent_; - - std::atomic last_picked_index_; - std::vector> subchannels_; - }; - - void ShutdownLocked() override; - - // List of subchannels. - RefCountedPtr subchannel_list_; - // Latest pending subchannel list. - // When we get an updated address list, we create a new subchannel list - // for it here, and we wait to swap it into subchannel_list_ until the new - // list becomes READY. - RefCountedPtr latest_pending_subchannel_list_; - - bool shutdown_ = false; - - absl::BitGen bit_gen_; -}; - -// -// OldRoundRobin::Picker -// - -OldRoundRobin::Picker::Picker(OldRoundRobin* parent, - RoundRobinSubchannelList* subchannel_list) - : parent_(parent) { - for (size_t i = 0; i < subchannel_list->num_subchannels(); ++i) { - RoundRobinSubchannelData* sd = subchannel_list->subchannel(i); - if (sd->connectivity_state().value_or(GRPC_CHANNEL_IDLE) == - GRPC_CHANNEL_READY) { - subchannels_.push_back(sd->subchannel()->Ref()); - } - } - // For discussion on why we generate a random starting index for - // the picker, see https://github.com/grpc/grpc-go/issues/2580. - size_t index = - absl::Uniform(parent->bit_gen_, 0, subchannels_.size()); - last_picked_index_.store(index, std::memory_order_relaxed); - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) { - gpr_log(GPR_INFO, - "[RR %p picker %p] created picker from subchannel_list=%p " - "with %" PRIuPTR " READY subchannels; last_picked_index_=%" PRIuPTR, - parent_, this, subchannel_list, subchannels_.size(), index); - } -} - -OldRoundRobin::PickResult OldRoundRobin::Picker::Pick(PickArgs /*args*/) { - size_t index = last_picked_index_.fetch_add(1, std::memory_order_relaxed) % - subchannels_.size(); - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) { - gpr_log(GPR_INFO, - "[RR %p picker %p] returning index %" PRIuPTR ", subchannel=%p", - parent_, this, index, subchannels_[index].get()); - } - return PickResult::Complete(subchannels_[index]); -} - -// -// RoundRobin -// - -OldRoundRobin::OldRoundRobin(Args args) : LoadBalancingPolicy(std::move(args)) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) { - gpr_log(GPR_INFO, "[RR %p] Created", this); - } -} - -OldRoundRobin::~OldRoundRobin() { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) { - gpr_log(GPR_INFO, "[RR %p] Destroying Round Robin policy", this); - } - GPR_ASSERT(subchannel_list_ == nullptr); - GPR_ASSERT(latest_pending_subchannel_list_ == nullptr); -} - -void OldRoundRobin::ShutdownLocked() { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) { - gpr_log(GPR_INFO, "[RR %p] Shutting down", this); - } - shutdown_ = true; - subchannel_list_.reset(); - latest_pending_subchannel_list_.reset(); -} - -void OldRoundRobin::ResetBackoffLocked() { - subchannel_list_->ResetBackoffLocked(); - if (latest_pending_subchannel_list_ != nullptr) { - latest_pending_subchannel_list_->ResetBackoffLocked(); - } -} - -absl::Status OldRoundRobin::UpdateLocked(UpdateArgs args) { - EndpointAddressesIterator* addresses = nullptr; - if (args.addresses.ok()) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) { - gpr_log(GPR_INFO, "[RR %p] received update", this); - } - addresses = args.addresses->get(); - } else { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) { - gpr_log(GPR_INFO, "[RR %p] received update with address error: %s", this, - args.addresses.status().ToString().c_str()); - } - // If we already have a subchannel list, then keep using the existing - // list, but still report back that the update was not accepted. - if (subchannel_list_ != nullptr) return args.addresses.status(); - } - // Create new subchannel list, replacing the previous pending list, if any. - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace) && - latest_pending_subchannel_list_ != nullptr) { - gpr_log(GPR_INFO, "[RR %p] replacing previous pending subchannel list %p", - this, latest_pending_subchannel_list_.get()); - } - latest_pending_subchannel_list_ = - MakeRefCounted(this, addresses, args.args); - latest_pending_subchannel_list_->StartWatchingLocked(args.args); - // If the new list is empty, immediately promote it to - // subchannel_list_ and report TRANSIENT_FAILURE. - if (latest_pending_subchannel_list_->num_subchannels() == 0) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace) && - subchannel_list_ != nullptr) { - gpr_log(GPR_INFO, "[RR %p] replacing previous subchannel list %p", this, - subchannel_list_.get()); - } - subchannel_list_ = std::move(latest_pending_subchannel_list_); - absl::Status status = - args.addresses.ok() ? absl::UnavailableError(absl::StrCat( - "empty address list: ", args.resolution_note)) - : args.addresses.status(); - channel_control_helper()->UpdateState( - GRPC_CHANNEL_TRANSIENT_FAILURE, status, - MakeRefCounted(status)); - return status; - } - // Otherwise, if this is the initial update, immediately promote it to - // subchannel_list_. - if (subchannel_list_.get() == nullptr) { - subchannel_list_ = std::move(latest_pending_subchannel_list_); - } - return absl::OkStatus(); -} - -// -// RoundRobinSubchannelList -// - -void OldRoundRobin::RoundRobinSubchannelList::UpdateStateCountersLocked( - absl::optional old_state, - grpc_connectivity_state new_state) { - if (old_state.has_value()) { - GPR_ASSERT(*old_state != GRPC_CHANNEL_SHUTDOWN); - if (*old_state == GRPC_CHANNEL_READY) { - GPR_ASSERT(num_ready_ > 0); - --num_ready_; - } else if (*old_state == GRPC_CHANNEL_CONNECTING) { - GPR_ASSERT(num_connecting_ > 0); - --num_connecting_; - } else if (*old_state == GRPC_CHANNEL_TRANSIENT_FAILURE) { - GPR_ASSERT(num_transient_failure_ > 0); - --num_transient_failure_; - } - } - GPR_ASSERT(new_state != GRPC_CHANNEL_SHUTDOWN); - if (new_state == GRPC_CHANNEL_READY) { - ++num_ready_; - } else if (new_state == GRPC_CHANNEL_CONNECTING) { - ++num_connecting_; - } else if (new_state == GRPC_CHANNEL_TRANSIENT_FAILURE) { - ++num_transient_failure_; - } -} - -void OldRoundRobin::RoundRobinSubchannelList:: - MaybeUpdateRoundRobinConnectivityStateLocked(absl::Status status_for_tf) { - OldRoundRobin* p = static_cast(policy()); - // If this is latest_pending_subchannel_list_, then swap it into - // subchannel_list_ in the following cases: - // - subchannel_list_ has no READY subchannels. - // - This list has at least one READY subchannel and we have seen the - // initial connectivity state notification for all subchannels. - // - All of the subchannels in this list are in TRANSIENT_FAILURE. - // (This may cause the channel to go from READY to TRANSIENT_FAILURE, - // but we're doing what the control plane told us to do.) - if (p->latest_pending_subchannel_list_.get() == this && - (p->subchannel_list_->num_ready_ == 0 || - (num_ready_ > 0 && AllSubchannelsSeenInitialState()) || - num_transient_failure_ == num_subchannels())) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) { - const std::string old_counters_string = - p->subchannel_list_ != nullptr ? p->subchannel_list_->CountersString() - : ""; - gpr_log( - GPR_INFO, - "[RR %p] swapping out subchannel list %p (%s) in favor of %p (%s)", p, - p->subchannel_list_.get(), old_counters_string.c_str(), this, - CountersString().c_str()); - } - p->subchannel_list_ = std::move(p->latest_pending_subchannel_list_); - } - // Only set connectivity state if this is the current subchannel list. - if (p->subchannel_list_.get() != this) return; - // First matching rule wins: - // 1) ANY subchannel is READY => policy is READY. - // 2) ANY subchannel is CONNECTING => policy is CONNECTING. - // 3) ALL subchannels are TRANSIENT_FAILURE => policy is TRANSIENT_FAILURE. - if (num_ready_ > 0) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) { - gpr_log(GPR_INFO, "[RR %p] reporting READY with subchannel list %p", p, - this); - } - p->channel_control_helper()->UpdateState(GRPC_CHANNEL_READY, absl::Status(), - MakeRefCounted(p, this)); - } else if (num_connecting_ > 0) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) { - gpr_log(GPR_INFO, "[RR %p] reporting CONNECTING with subchannel list %p", - p, this); - } - p->channel_control_helper()->UpdateState( - GRPC_CHANNEL_CONNECTING, absl::Status(), - MakeRefCounted( - p->RefAsSubclass(DEBUG_LOCATION, "QueuePicker"))); - } else if (num_transient_failure_ == num_subchannels()) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) { - gpr_log(GPR_INFO, - "[RR %p] reporting TRANSIENT_FAILURE with subchannel list %p: %s", - p, this, status_for_tf.ToString().c_str()); - } - if (!status_for_tf.ok()) { - last_failure_ = absl::UnavailableError( - absl::StrCat("connections to all backends failing; last error: ", - status_for_tf.ToString())); - } - p->channel_control_helper()->UpdateState( - GRPC_CHANNEL_TRANSIENT_FAILURE, last_failure_, - MakeRefCounted(last_failure_)); - } -} - -// -// RoundRobinSubchannelData -// - -void OldRoundRobin::RoundRobinSubchannelData::ProcessConnectivityChangeLocked( - absl::optional old_state, - grpc_connectivity_state new_state) { - OldRoundRobin* p = static_cast(subchannel_list()->policy()); - GPR_ASSERT(subchannel() != nullptr); - // If this is not the initial state notification and the new state is - // TRANSIENT_FAILURE or IDLE, re-resolve. - // Note that we don't want to do this on the initial state notification, - // because that would result in an endless loop of re-resolution. - if (old_state.has_value() && (new_state == GRPC_CHANNEL_TRANSIENT_FAILURE || - new_state == GRPC_CHANNEL_IDLE)) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) { - gpr_log(GPR_INFO, - "[RR %p] Subchannel %p reported %s; requesting re-resolution", p, - subchannel(), ConnectivityStateName(new_state)); - } - p->channel_control_helper()->RequestReresolution(); - } - if (new_state == GRPC_CHANNEL_IDLE) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) { - gpr_log(GPR_INFO, - "[RR %p] Subchannel %p reported IDLE; requesting connection", p, - subchannel()); - } - subchannel()->RequestConnection(); - } - // Update logical connectivity state. - UpdateLogicalConnectivityStateLocked(new_state); - // Update the policy state. - subchannel_list()->MaybeUpdateRoundRobinConnectivityStateLocked( - connectivity_status()); -} - -void OldRoundRobin::RoundRobinSubchannelData:: - UpdateLogicalConnectivityStateLocked( - grpc_connectivity_state connectivity_state) { - OldRoundRobin* p = static_cast(subchannel_list()->policy()); - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) { - gpr_log( - GPR_INFO, - "[RR %p] connectivity changed for subchannel %p, subchannel_list %p " - "(index %" PRIuPTR " of %" PRIuPTR "): prev_state=%s new_state=%s", - p, subchannel(), subchannel_list(), Index(), - subchannel_list()->num_subchannels(), - (logical_connectivity_state_.has_value() - ? ConnectivityStateName(*logical_connectivity_state_) - : "N/A"), - ConnectivityStateName(connectivity_state)); - } - // Decide what state to report for aggregation purposes. - // If the last logical state was TRANSIENT_FAILURE, then ignore the - // state change unless the new state is READY. - if (logical_connectivity_state_.has_value() && - *logical_connectivity_state_ == GRPC_CHANNEL_TRANSIENT_FAILURE && - connectivity_state != GRPC_CHANNEL_READY) { - return; - } - // If the new state is IDLE, treat it as CONNECTING, since it will - // immediately transition into CONNECTING anyway. - if (connectivity_state == GRPC_CHANNEL_IDLE) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) { - gpr_log(GPR_INFO, - "[RR %p] subchannel %p, subchannel_list %p (index %" PRIuPTR - " of %" PRIuPTR "): treating IDLE as CONNECTING", - p, subchannel(), subchannel_list(), Index(), - subchannel_list()->num_subchannels()); - } - connectivity_state = GRPC_CHANNEL_CONNECTING; - } - // If no change, return false. - if (logical_connectivity_state_.has_value() && - *logical_connectivity_state_ == connectivity_state) { - return; - } - // Otherwise, update counters and logical state. - subchannel_list()->UpdateStateCountersLocked(logical_connectivity_state_, - connectivity_state); - logical_connectivity_state_ = connectivity_state; -} - -// -// round_robin LB policy (with dualstack changes) -// - class RoundRobin : public LoadBalancingPolicy { public: explicit RoundRobin(Args args); @@ -892,9 +440,6 @@ class RoundRobinFactory : public LoadBalancingPolicyFactory { public: OrphanablePtr CreateLoadBalancingPolicy( LoadBalancingPolicy::Args args) const override { - if (!IsRoundRobinDelegateToPickFirstEnabled()) { - return MakeOrphanable(std::move(args)); - } return MakeOrphanable(std::move(args)); } diff --git a/src/core/load_balancing/subchannel_list.h b/src/core/load_balancing/subchannel_list.h deleted file mode 100644 index da9e35272a6..00000000000 --- a/src/core/load_balancing/subchannel_list.h +++ /dev/null @@ -1,455 +0,0 @@ -// -// Copyright 2015 gRPC authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -#ifndef GRPC_SRC_CORE_LOAD_BALANCING_SUBCHANNEL_LIST_H -#define GRPC_SRC_CORE_LOAD_BALANCING_SUBCHANNEL_LIST_H - -#include - -#include -#include - -#include -#include -#include - -#include "absl/status/status.h" -#include "absl/types/optional.h" - -#include -#include - -#include "src/core/load_balancing/health_check_client.h" -#include "src/core/lib/channel/channel_args.h" -#include "src/core/lib/gprpp/debug_location.h" -#include "src/core/lib/gprpp/dual_ref_counted.h" -#include "src/core/lib/gprpp/manual_constructor.h" -#include "src/core/lib/gprpp/ref_counted_ptr.h" -#include "src/core/lib/gprpp/work_serializer.h" -#include "src/core/lib/iomgr/iomgr_fwd.h" -#include "src/core/lib/transport/connectivity_state.h" -#include "src/core/load_balancing/lb_policy.h" -#include "src/core/load_balancing/subchannel_interface.h" -#include "src/core/resolver/endpoint_addresses.h" -#include "src/core/resolver/server_address.h" - -// Code for maintaining a list of subchannels within an LB policy. -// -// To use this, callers must create their own subclasses, like so: -// - -// class MySubchannelList; // Forward declaration. - -// class MySubchannelData -// : public SubchannelData { -// public: -// void ProcessConnectivityChangeLocked( -// absl::optional old_state, -// grpc_connectivity_state new_state) override { -// // ...code to handle connectivity changes... -// } -// }; - -// class MySubchannelList -// : public SubchannelList { -// }; - -// -// All methods will be called from within the client_channel work serializer. - -namespace grpc_core { - -// Forward declaration. -template -class SubchannelList; - -// Stores data for a particular subchannel in a subchannel list. -// Callers must create a subclass that implements the -// ProcessConnectivityChangeLocked() method. -template -class SubchannelData { - public: - // Returns a pointer to the subchannel list containing this object. - SubchannelListType* subchannel_list() const { - return static_cast(subchannel_list_); - } - - // Returns the index into the subchannel list of this object. - size_t Index() const { - return static_cast(static_cast(this) - - subchannel_list_->subchannel(0)); - } - - // Returns a pointer to the subchannel. - SubchannelInterface* subchannel() const { return subchannel_.get(); } - - // Returns the cached connectivity state, if any. - absl::optional connectivity_state() { - return connectivity_state_; - } - absl::Status connectivity_status() { return connectivity_status_; } - - // Resets the connection backoff. - void ResetBackoffLocked(); - - // Cancels any pending connectivity watch and unrefs the subchannel. - void ShutdownLocked(); - - protected: - SubchannelData( - SubchannelList* subchannel_list, - const ServerAddress& address, - RefCountedPtr subchannel); - - virtual ~SubchannelData(); - - // This method will be invoked once soon after instantiation to report - // the current connectivity state, and it will then be invoked again - // whenever the connectivity state changes. - virtual void ProcessConnectivityChangeLocked( - absl::optional old_state, - grpc_connectivity_state new_state) = 0; - - private: - // For accessing StartConnectivityWatchLocked(). - friend class SubchannelList; - - // Watcher for subchannel connectivity state. - class Watcher - : public SubchannelInterface::ConnectivityStateWatcherInterface { - public: - Watcher( - SubchannelData* subchannel_data, - WeakRefCountedPtr subchannel_list) - : subchannel_data_(subchannel_data), - subchannel_list_(std::move(subchannel_list)) {} - - ~Watcher() override { - subchannel_list_.reset(DEBUG_LOCATION, "Watcher dtor"); - } - - void OnConnectivityStateChange(grpc_connectivity_state new_state, - absl::Status status) override; - - grpc_pollset_set* interested_parties() override { - return subchannel_list_->policy()->interested_parties(); - } - - private: - SubchannelData* subchannel_data_; - WeakRefCountedPtr subchannel_list_; - }; - - // Starts watching the connectivity state of the subchannel. - // ProcessConnectivityChangeLocked() will be called whenever the - // connectivity state changes. - void StartConnectivityWatchLocked(const ChannelArgs& args); - - // Cancels watching the connectivity state of the subchannel. - void CancelConnectivityWatchLocked(const char* reason); - - // Unrefs the subchannel. - void UnrefSubchannelLocked(const char* reason); - - // Backpointer to owning subchannel list. Not owned. - SubchannelList* subchannel_list_; - // The subchannel. - RefCountedPtr subchannel_; - // Will be non-null when the subchannel's state is being watched. - SubchannelInterface::DataWatcherInterface* health_watcher_ = nullptr; - // Data updated by the watcher. - absl::optional connectivity_state_; - absl::Status connectivity_status_; -}; - -// A list of subchannels. -template -class SubchannelList : public DualRefCounted { - public: - // Starts watching the connectivity state of all subchannels. - // Must be called immediately after instantiation. - void StartWatchingLocked(const ChannelArgs& args); - - // The number of subchannels in the list. - size_t num_subchannels() const { return subchannels_.size(); } - - // The data for the subchannel at a particular index. - SubchannelDataType* subchannel(size_t index) { - return subchannels_[index].get(); - } - - // Returns true if the subchannel list is shutting down. - bool shutting_down() const { return shutting_down_; } - - // Accessors. - LoadBalancingPolicy* policy() const { return policy_; } - const char* tracer() const { return tracer_; } - - // Resets connection backoff of all subchannels. - void ResetBackoffLocked(); - - // Returns true if all subchannels have seen their initial - // connectivity state notifications. - bool AllSubchannelsSeenInitialState(); - - void Orphan() override; - - protected: - SubchannelList(LoadBalancingPolicy* policy, const char* tracer, - EndpointAddressesIterator* addresses, - LoadBalancingPolicy::ChannelControlHelper* helper, - const ChannelArgs& args); - - virtual ~SubchannelList(); - - private: - // For accessing Ref() and Unref(). - friend class SubchannelData; - - virtual std::shared_ptr work_serializer() const = 0; - - // Backpointer to owning policy. - LoadBalancingPolicy* policy_; - - const char* tracer_; - - // The list of subchannels. - // We use ManualConstructor here to support SubchannelDataType classes - // that are not copyable. - std::vector> subchannels_; - - // Is this list shutting down? This may be true due to the shutdown of the - // policy itself or because a newer update has arrived while this one hadn't - // finished processing. - bool shutting_down_ = false; -}; - -// -// implementation -- no user-servicable parts below -// - -// -// SubchannelData::Watcher -// - -template -void SubchannelData::Watcher:: - OnConnectivityStateChange(grpc_connectivity_state new_state, - absl::Status status) { - if (GPR_UNLIKELY(subchannel_list_->tracer() != nullptr)) { - gpr_log( - GPR_INFO, - "[%s %p] subchannel list %p index %" PRIuPTR " of %" PRIuPTR - " (subchannel %p): connectivity changed: old_state=%s, new_state=%s, " - "status=%s, shutting_down=%d, health_watcher=%p", - subchannel_list_->tracer(), subchannel_list_->policy(), - subchannel_list_.get(), subchannel_data_->Index(), - subchannel_list_->num_subchannels(), - subchannel_data_->subchannel_.get(), - (subchannel_data_->connectivity_state_.has_value() - ? ConnectivityStateName(*subchannel_data_->connectivity_state_) - : "N/A"), - ConnectivityStateName(new_state), status.ToString().c_str(), - subchannel_list_->shutting_down(), subchannel_data_->health_watcher_); - } - if (!subchannel_list_->shutting_down() && - subchannel_data_->health_watcher_ != nullptr) { - absl::optional old_state = - subchannel_data_->connectivity_state_; - subchannel_data_->connectivity_state_ = new_state; - subchannel_data_->connectivity_status_ = status; - // Call the subclass's ProcessConnectivityChangeLocked() method. - subchannel_data_->ProcessConnectivityChangeLocked(old_state, new_state); - } -} - -// -// SubchannelData -// - -template -SubchannelData::SubchannelData( - SubchannelList* subchannel_list, - const ServerAddress& /*address*/, - RefCountedPtr subchannel) - : subchannel_list_(subchannel_list), subchannel_(std::move(subchannel)) {} - -template -SubchannelData::~SubchannelData() { - GPR_ASSERT(subchannel_ == nullptr); -} - -template -void SubchannelData:: - UnrefSubchannelLocked(const char* reason) { - if (subchannel_ != nullptr) { - if (GPR_UNLIKELY(subchannel_list_->tracer() != nullptr)) { - gpr_log(GPR_INFO, - "[%s %p] subchannel list %p index %" PRIuPTR " of %" PRIuPTR - " (subchannel %p): unreffing subchannel (%s)", - subchannel_list_->tracer(), subchannel_list_->policy(), - subchannel_list_, Index(), subchannel_list_->num_subchannels(), - subchannel_.get(), reason); - } - subchannel_.reset(); - } -} - -template -void SubchannelData::ResetBackoffLocked() { - if (subchannel_ != nullptr) { - subchannel_->ResetBackoff(); - } -} - -template -void SubchannelData:: - StartConnectivityWatchLocked(const ChannelArgs& args) { - if (GPR_UNLIKELY(subchannel_list_->tracer() != nullptr)) { - gpr_log(GPR_INFO, - "[%s %p] subchannel list %p index %" PRIuPTR " of %" PRIuPTR - " (subchannel %p): starting watch", - subchannel_list_->tracer(), subchannel_list_->policy(), - subchannel_list_, Index(), subchannel_list_->num_subchannels(), - subchannel_.get()); - } - GPR_ASSERT(health_watcher_ == nullptr); - auto watcher = std::make_unique( - this, subchannel_list()->WeakRef(DEBUG_LOCATION, "Watcher")); - auto health_watcher = MakeHealthCheckWatcher( - subchannel_list_->work_serializer(), args, std::move(watcher)); - health_watcher_ = health_watcher.get(); - subchannel_->AddDataWatcher(std::move(health_watcher)); -} - -template -void SubchannelData:: - CancelConnectivityWatchLocked(const char* reason) { - if (health_watcher_ != nullptr) { - if (GPR_UNLIKELY(subchannel_list_->tracer() != nullptr)) { - gpr_log(GPR_INFO, - "[%s %p] subchannel list %p index %" PRIuPTR " of %" PRIuPTR - " (subchannel %p): canceling health watch (%s)", - subchannel_list_->tracer(), subchannel_list_->policy(), - subchannel_list_, Index(), subchannel_list_->num_subchannels(), - subchannel_.get(), reason); - } - subchannel_->CancelDataWatcher(health_watcher_); - health_watcher_ = nullptr; - } -} - -template -void SubchannelData::ShutdownLocked() { - CancelConnectivityWatchLocked("shutdown"); - UnrefSubchannelLocked("shutdown"); -} - -// -// SubchannelList -// - -template -SubchannelList::SubchannelList( - LoadBalancingPolicy* policy, const char* tracer, - EndpointAddressesIterator* addresses, - LoadBalancingPolicy::ChannelControlHelper* helper, const ChannelArgs& args) - : DualRefCounted(tracer), - policy_(policy), - tracer_(tracer) { - if (GPR_UNLIKELY(tracer_ != nullptr)) { - gpr_log(GPR_INFO, "[%s %p] Creating subchannel list %p", tracer_, policy, - this); - } - if (addresses == nullptr) return; - // Create a subchannel for each address. - addresses->ForEach([&](const EndpointAddresses& address) { - RefCountedPtr subchannel = - helper->CreateSubchannel(address.address(), address.args(), args); - if (subchannel == nullptr) { - // Subchannel could not be created. - if (GPR_UNLIKELY(tracer_ != nullptr)) { - gpr_log(GPR_INFO, - "[%s %p] could not create subchannel for address %s, ignoring", - tracer_, policy_, address.ToString().c_str()); - } - return; - } - if (GPR_UNLIKELY(tracer_ != nullptr)) { - gpr_log(GPR_INFO, - "[%s %p] subchannel list %p index %" PRIuPTR - ": Created subchannel %p for address %s", - tracer_, policy_, this, subchannels_.size(), subchannel.get(), - address.ToString().c_str()); - } - subchannels_.emplace_back(); - subchannels_.back().Init(this, address, std::move(subchannel)); - }); -} - -template -SubchannelList::~SubchannelList() { - if (GPR_UNLIKELY(tracer_ != nullptr)) { - gpr_log(GPR_INFO, "[%s %p] Destroying subchannel_list %p", tracer_, policy_, - this); - } - for (auto& sd : subchannels_) { - sd.Destroy(); - } -} - -template -void SubchannelList:: - StartWatchingLocked(const ChannelArgs& args) { - for (auto& sd : subchannels_) { - sd->StartConnectivityWatchLocked(args); - } -} - -template -void SubchannelList::Orphan() { - if (GPR_UNLIKELY(tracer_ != nullptr)) { - gpr_log(GPR_INFO, "[%s %p] Shutting down subchannel_list %p", tracer_, - policy_, this); - } - GPR_ASSERT(!shutting_down_); - shutting_down_ = true; - for (auto& sd : subchannels_) { - sd->ShutdownLocked(); - } -} - -template -void SubchannelList::ResetBackoffLocked() { - for (auto& sd : subchannels_) { - sd->ResetBackoffLocked(); - } -} - -template -bool SubchannelList::AllSubchannelsSeenInitialState() { - for (size_t i = 0; i < num_subchannels(); ++i) { - if (!subchannel(i)->connectivity_state().has_value()) return false; - } - return true; -} - -} // namespace grpc_core - -#endif // GRPC_SRC_CORE_LOAD_BALANCING_SUBCHANNEL_LIST_H diff --git a/src/core/load_balancing/weighted_round_robin/weighted_round_robin.cc b/src/core/load_balancing/weighted_round_robin/weighted_round_robin.cc index 9a049a5a03e..14f01fb267e 100644 --- a/src/core/load_balancing/weighted_round_robin/weighted_round_robin.cc +++ b/src/core/load_balancing/weighted_round_robin/weighted_round_robin.cc @@ -18,11 +18,9 @@ #include #include -#include #include #include -#include #include #include #include @@ -46,13 +44,6 @@ #include #include -#include "src/core/load_balancing/backend_metric_data.h" -#include "src/core/load_balancing/endpoint_list.h" -#include "src/core/load_balancing/oob_backend_metric.h" -#include "src/core/load_balancing/subchannel_list.h" -#include "src/core/load_balancing/weighted_round_robin/static_stride_scheduler.h" -#include "src/core/load_balancing/weighted_target/weighted_target.h" -#include "src/core/lib/address_utils/sockaddr_utils.h" #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/channel/metrics.h" #include "src/core/lib/config/core_configuration.h" @@ -72,970 +63,123 @@ #include "src/core/lib/iomgr/resolved_address.h" #include "src/core/lib/json/json.h" #include "src/core/lib/json/json_args.h" -#include "src/core/lib/json/json_object_loader.h" -#include "src/core/lib/transport/connectivity_state.h" -#include "src/core/load_balancing/lb_policy.h" -#include "src/core/load_balancing/lb_policy_factory.h" -#include "src/core/load_balancing/subchannel_interface.h" -#include "src/core/resolver/endpoint_addresses.h" -#include "src/core/resolver/server_address.h" - -namespace grpc_core { - -TraceFlag grpc_lb_wrr_trace(false, "weighted_round_robin_lb"); - -namespace { - -constexpr absl::string_view kWeightedRoundRobin = "weighted_round_robin"; - -constexpr absl::string_view kMetricLabelLocality = "grpc.lb.locality"; - -const auto kMetricRrFallback = - GlobalInstrumentsRegistry::RegisterUInt64Counter( - "grpc.lb.wrr.rr_fallback", - "EXPERIMENTAL. Number of scheduler updates in which there were not " - "enough endpoints with valid weight, which caused the WRR policy to " - "fall back to RR behavior.", - "{update}", {kMetricLabelTarget}, {kMetricLabelLocality}, false); - -const auto kMetricEndpointWeightNotYetUsable = - GlobalInstrumentsRegistry::RegisterUInt64Counter( - "grpc.lb.wrr.endpoint_weight_not_yet_usable", - "EXPERIMENTAL. Number of endpoints from each scheduler update that " - "don't yet have usable weight information (i.e., either the load " - "report has not yet been received, or it is within the blackout " - "period).", - "{endpoint}", {kMetricLabelTarget}, {kMetricLabelLocality}, false); - -const auto kMetricEndpointWeightStale = - GlobalInstrumentsRegistry::RegisterUInt64Counter( - "grpc.lb.wrr.endpoint_weight_stale", - "EXPERIMENTAL. Number of endpoints from each scheduler update whose " - "latest weight is older than the expiration period.", - "{endpoint}", {kMetricLabelTarget}, {kMetricLabelLocality}, false); - -const auto kMetricEndpointWeights = - GlobalInstrumentsRegistry::RegisterDoubleHistogram( - "grpc.lb.wrr.endpoint_weights", - "EXPERIMENTAL. The histogram buckets will be endpoint weight ranges. " - "Each bucket will be a counter that is incremented once for every " - "endpoint whose weight is within that range. Note that endpoints " - "without usable weights will have weight 0.", - "{weight}", {kMetricLabelTarget}, {kMetricLabelLocality}, false); - -// Config for WRR policy. -class WeightedRoundRobinConfig : public LoadBalancingPolicy::Config { - public: - WeightedRoundRobinConfig() = default; - - WeightedRoundRobinConfig(const WeightedRoundRobinConfig&) = delete; - WeightedRoundRobinConfig& operator=(const WeightedRoundRobinConfig&) = delete; - - WeightedRoundRobinConfig(WeightedRoundRobinConfig&&) = delete; - WeightedRoundRobinConfig& operator=(WeightedRoundRobinConfig&&) = delete; - - absl::string_view name() const override { return kWeightedRoundRobin; } - - bool enable_oob_load_report() const { return enable_oob_load_report_; } - Duration oob_reporting_period() const { return oob_reporting_period_; } - Duration blackout_period() const { return blackout_period_; } - Duration weight_update_period() const { return weight_update_period_; } - Duration weight_expiration_period() const { - return weight_expiration_period_; - } - float error_utilization_penalty() const { return error_utilization_penalty_; } - - static const JsonLoaderInterface* JsonLoader(const JsonArgs&) { - static const auto* loader = - JsonObjectLoader() - .OptionalField("enableOobLoadReport", - &WeightedRoundRobinConfig::enable_oob_load_report_) - .OptionalField("oobReportingPeriod", - &WeightedRoundRobinConfig::oob_reporting_period_) - .OptionalField("blackoutPeriod", - &WeightedRoundRobinConfig::blackout_period_) - .OptionalField("weightUpdatePeriod", - &WeightedRoundRobinConfig::weight_update_period_) - .OptionalField("weightExpirationPeriod", - &WeightedRoundRobinConfig::weight_expiration_period_) - .OptionalField( - "errorUtilizationPenalty", - &WeightedRoundRobinConfig::error_utilization_penalty_) - .Finish(); - return loader; - } - - void JsonPostLoad(const Json&, const JsonArgs&, ValidationErrors* errors) { - // Impose lower bound of 100ms on weightUpdatePeriod. - weight_update_period_ = - std::max(weight_update_period_, Duration::Milliseconds(100)); - if (error_utilization_penalty_ < 0) { - ValidationErrors::ScopedField field(errors, ".errorUtilizationPenalty"); - errors->AddError("must be non-negative"); - } - } - - private: - bool enable_oob_load_report_ = false; - Duration oob_reporting_period_ = Duration::Seconds(10); - Duration blackout_period_ = Duration::Seconds(10); - Duration weight_update_period_ = Duration::Seconds(1); - Duration weight_expiration_period_ = Duration::Minutes(3); - float error_utilization_penalty_ = 1.0; -}; - -// Legacy WRR LB policy (not delegating to pick_first) -class OldWeightedRoundRobin : public LoadBalancingPolicy { - public: - explicit OldWeightedRoundRobin(Args args); - - absl::string_view name() const override { return kWeightedRoundRobin; } - - absl::Status UpdateLocked(UpdateArgs args) override; - void ResetBackoffLocked() override; - - private: - // Represents the weight for a given address. - class AddressWeight : public RefCounted { - public: - AddressWeight(RefCountedPtr wrr, std::string key) - : wrr_(std::move(wrr)), key_(std::move(key)) {} - ~AddressWeight() override; - - void MaybeUpdateWeight(double qps, double eps, double utilization, - float error_utilization_penalty); - - float GetWeight(Timestamp now, Duration weight_expiration_period, - Duration blackout_period); - - void ResetNonEmptySince(); - - private: - RefCountedPtr wrr_; - const std::string key_; - - Mutex mu_; - float weight_ ABSL_GUARDED_BY(&mu_) = 0; - Timestamp non_empty_since_ ABSL_GUARDED_BY(&mu_) = Timestamp::InfFuture(); - Timestamp last_update_time_ ABSL_GUARDED_BY(&mu_) = Timestamp::InfPast(); - }; - - // Forward declaration. - class WeightedRoundRobinSubchannelList; - - // Data for a particular subchannel in a subchannel list. - // This subclass adds the following functionality: - // - Tracks the previous connectivity state of the subchannel, so that - // we know how many subchannels are in each state. - class WeightedRoundRobinSubchannelData - : public SubchannelData { - public: - WeightedRoundRobinSubchannelData( - SubchannelList* subchannel_list, - const ServerAddress& address, RefCountedPtr sc); - - absl::optional connectivity_state() const { - return logical_connectivity_state_; - } - - RefCountedPtr weight() const { return weight_; } - - private: - class OobWatcher : public OobBackendMetricWatcher { - public: - OobWatcher(RefCountedPtr weight, - float error_utilization_penalty) - : weight_(std::move(weight)), - error_utilization_penalty_(error_utilization_penalty) {} - - void OnBackendMetricReport( - const BackendMetricData& backend_metric_data) override; - - private: - RefCountedPtr weight_; - const float error_utilization_penalty_; - }; - - // Performs connectivity state updates that need to be done only - // after we have started watching. - void ProcessConnectivityChangeLocked( - absl::optional old_state, - grpc_connectivity_state new_state) override; - - // Updates the logical connectivity state. - void UpdateLogicalConnectivityStateLocked( - grpc_connectivity_state connectivity_state); - - // The logical connectivity state of the subchannel. - // Note that the logical connectivity state may differ from the - // actual reported state in some cases (e.g., after we see - // TRANSIENT_FAILURE, we ignore any subsequent state changes until - // we see READY). - absl::optional logical_connectivity_state_; - - RefCountedPtr weight_; - }; - - // A list of subchannels. - class WeightedRoundRobinSubchannelList - : public SubchannelList { - public: - WeightedRoundRobinSubchannelList(OldWeightedRoundRobin* policy, - EndpointAddressesIterator* addresses, - const ChannelArgs& args) - : SubchannelList(policy, - (GRPC_TRACE_FLAG_ENABLED(grpc_lb_wrr_trace) - ? "WeightedRoundRobinSubchannelList" - : nullptr), - addresses, policy->channel_control_helper(), args) { - // Need to maintain a ref to the LB policy as long as we maintain - // any references to subchannels, since the subchannels' - // pollset_sets will include the LB policy's pollset_set. - policy->Ref(DEBUG_LOCATION, "subchannel_list").release(); - } - - ~WeightedRoundRobinSubchannelList() override { - OldWeightedRoundRobin* p = static_cast(policy()); - p->Unref(DEBUG_LOCATION, "subchannel_list"); - } - - // Updates the counters of subchannels in each state when a - // subchannel transitions from old_state to new_state. - void UpdateStateCountersLocked( - absl::optional old_state, - grpc_connectivity_state new_state); - - // Ensures that the right subchannel list is used and then updates - // the aggregated connectivity state based on the subchannel list's - // state counters. - void MaybeUpdateAggregatedConnectivityStateLocked( - absl::Status status_for_tf); - - private: - std::shared_ptr work_serializer() const override { - return static_cast(policy())->work_serializer(); - } - - std::string CountersString() const { - return absl::StrCat("num_subchannels=", num_subchannels(), - " num_ready=", num_ready_, - " num_connecting=", num_connecting_, - " num_transient_failure=", num_transient_failure_); - } - - size_t num_ready_ = 0; - size_t num_connecting_ = 0; - size_t num_transient_failure_ = 0; - - absl::Status last_failure_; - }; - - // A picker that performs WRR picks with weights based on - // endpoint-reported utilization and QPS. - class Picker : public SubchannelPicker { - public: - Picker(RefCountedPtr wrr, - WeightedRoundRobinSubchannelList* subchannel_list); - - ~Picker() override; - - PickResult Pick(PickArgs args) override; - - void Orphan() override; - - private: - // A call tracker that collects per-call endpoint utilization reports. - class SubchannelCallTracker : public SubchannelCallTrackerInterface { - public: - SubchannelCallTracker(RefCountedPtr weight, - float error_utilization_penalty) - : weight_(std::move(weight)), - error_utilization_penalty_(error_utilization_penalty) {} - - void Start() override {} - - void Finish(FinishArgs args) override; - - private: - RefCountedPtr weight_; - const float error_utilization_penalty_; - }; - - // Info stored about each subchannel. - struct SubchannelInfo { - SubchannelInfo(RefCountedPtr subchannel, - RefCountedPtr weight) - : subchannel(std::move(subchannel)), weight(std::move(weight)) {} - - RefCountedPtr subchannel; - RefCountedPtr weight; - }; - - // Returns the index into subchannels_ to be picked. - size_t PickIndex(); - - // Builds a new scheduler and swaps it into place, then starts a - // timer for the next update. - void BuildSchedulerAndStartTimerLocked() - ABSL_EXCLUSIVE_LOCKS_REQUIRED(&timer_mu_); - - RefCountedPtr wrr_; - RefCountedPtr config_; - std::vector subchannels_; - - Mutex scheduler_mu_; - std::shared_ptr scheduler_ - ABSL_GUARDED_BY(&scheduler_mu_); - - Mutex timer_mu_ ABSL_ACQUIRED_BEFORE(&scheduler_mu_); - absl::optional - timer_handle_ ABSL_GUARDED_BY(&timer_mu_); - - // Used when falling back to RR. - std::atomic last_picked_index_; - }; - - ~OldWeightedRoundRobin() override; - - void ShutdownLocked() override; - - RefCountedPtr GetOrCreateWeight( - const grpc_resolved_address& address); - - RefCountedPtr config_; - - // List of subchannels. - RefCountedPtr subchannel_list_; - // Latest pending subchannel list. - // When we get an updated address list, we create a new subchannel list - // for it here, and we wait to swap it into subchannel_list_ until the new - // list becomes READY. - RefCountedPtr - latest_pending_subchannel_list_; - - Mutex address_weight_map_mu_; - std::map> address_weight_map_ - ABSL_GUARDED_BY(&address_weight_map_mu_); - - bool shutdown_ = false; - - absl::BitGen bit_gen_; - - // Accessed by picker. - std::atomic scheduler_state_{absl::Uniform(bit_gen_)}; -}; - -// -// OldWeightedRoundRobin::AddressWeight -// - -OldWeightedRoundRobin::AddressWeight::~AddressWeight() { - MutexLock lock(&wrr_->address_weight_map_mu_); - auto it = wrr_->address_weight_map_.find(key_); - if (it != wrr_->address_weight_map_.end() && it->second == this) { - wrr_->address_weight_map_.erase(it); - } -} - -void OldWeightedRoundRobin::AddressWeight::MaybeUpdateWeight( - double qps, double eps, double utilization, - float error_utilization_penalty) { - // Compute weight. - float weight = 0; - if (qps > 0 && utilization > 0) { - double penalty = 0.0; - if (eps > 0 && error_utilization_penalty > 0) { - penalty = eps / qps * error_utilization_penalty; - } - weight = qps / (utilization + penalty); - } - if (weight == 0) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_wrr_trace)) { - gpr_log(GPR_INFO, - "[WRR %p] subchannel %s: qps=%f, eps=%f, utilization=%f: " - "error_util_penalty=%f, weight=%f (not updating)", - wrr_.get(), key_.c_str(), qps, eps, utilization, - error_utilization_penalty, weight); - } - return; - } - Timestamp now = Timestamp::Now(); - // Grab the lock and update the data. - MutexLock lock(&mu_); - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_wrr_trace)) { - gpr_log(GPR_INFO, - "[WRR %p] subchannel %s: qps=%f, eps=%f, utilization=%f " - "error_util_penalty=%f : setting weight=%f weight_=%f now=%s " - "last_update_time_=%s non_empty_since_=%s", - wrr_.get(), key_.c_str(), qps, eps, utilization, - error_utilization_penalty, weight, weight_, now.ToString().c_str(), - last_update_time_.ToString().c_str(), - non_empty_since_.ToString().c_str()); - } - if (non_empty_since_ == Timestamp::InfFuture()) non_empty_since_ = now; - weight_ = weight; - last_update_time_ = now; -} - -float OldWeightedRoundRobin::AddressWeight::GetWeight( - Timestamp now, Duration weight_expiration_period, - Duration blackout_period) { - MutexLock lock(&mu_); - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_wrr_trace)) { - gpr_log(GPR_INFO, - "[WRR %p] subchannel %s: getting weight: now=%s " - "weight_expiration_period=%s blackout_period=%s " - "last_update_time_=%s non_empty_since_=%s weight_=%f", - wrr_.get(), key_.c_str(), now.ToString().c_str(), - weight_expiration_period.ToString().c_str(), - blackout_period.ToString().c_str(), - last_update_time_.ToString().c_str(), - non_empty_since_.ToString().c_str(), weight_); - } - // If the most recent update was longer ago than the expiration - // period, reset non_empty_since_ so that we apply the blackout period - // again if we start getting data again in the future, and return 0. - if (now - last_update_time_ >= weight_expiration_period) { - non_empty_since_ = Timestamp::InfFuture(); - return 0; - } - // If we don't have at least blackout_period worth of data, return 0. - if (blackout_period > Duration::Zero() && - now - non_empty_since_ < blackout_period) { - return 0; - } - // Otherwise, return the weight. - return weight_; -} - -void OldWeightedRoundRobin::AddressWeight::ResetNonEmptySince() { - MutexLock lock(&mu_); - non_empty_since_ = Timestamp::InfFuture(); -} - -// -// OldWeightedRoundRobin::Picker::SubchannelCallTracker -// - -void OldWeightedRoundRobin::Picker::SubchannelCallTracker::Finish( - FinishArgs args) { - auto* backend_metric_data = - args.backend_metric_accessor->GetBackendMetricData(); - double qps = 0; - double eps = 0; - double utilization = 0; - if (backend_metric_data != nullptr) { - qps = backend_metric_data->qps; - eps = backend_metric_data->eps; - utilization = backend_metric_data->application_utilization; - if (utilization <= 0) { - utilization = backend_metric_data->cpu_utilization; - } - } - weight_->MaybeUpdateWeight(qps, eps, utilization, error_utilization_penalty_); -} - -// -// OldWeightedRoundRobin::Picker -// - -OldWeightedRoundRobin::Picker::Picker( - RefCountedPtr wrr, - WeightedRoundRobinSubchannelList* subchannel_list) - : wrr_(std::move(wrr)), - config_(wrr_->config_), - last_picked_index_(absl::Uniform(wrr_->bit_gen_)) { - for (size_t i = 0; i < subchannel_list->num_subchannels(); ++i) { - WeightedRoundRobinSubchannelData* sd = subchannel_list->subchannel(i); - if (sd->connectivity_state() == GRPC_CHANNEL_READY) { - subchannels_.emplace_back(sd->subchannel()->Ref(), sd->weight()); - } - } - global_stats().IncrementWrrSubchannelListSize( - subchannel_list->num_subchannels()); - global_stats().IncrementWrrSubchannelReadySize(subchannels_.size()); - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_wrr_trace)) { - gpr_log(GPR_INFO, - "[WRR %p picker %p] created picker from subchannel_list=%p " - "with %" PRIuPTR " subchannels", - wrr_.get(), this, subchannel_list, subchannels_.size()); - } - BuildSchedulerAndStartTimerLocked(); -} - -OldWeightedRoundRobin::Picker::~Picker() { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_wrr_trace)) { - gpr_log(GPR_INFO, "[WRR %p picker %p] destroying picker", wrr_.get(), this); - } -} - -void OldWeightedRoundRobin::Picker::Orphan() { - MutexLock lock(&timer_mu_); - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_wrr_trace)) { - gpr_log(GPR_INFO, "[WRR %p picker %p] cancelling timer", wrr_.get(), this); - } - wrr_->channel_control_helper()->GetEventEngine()->Cancel(*timer_handle_); - timer_handle_.reset(); -} - -OldWeightedRoundRobin::PickResult OldWeightedRoundRobin::Picker::Pick( - PickArgs /*args*/) { - size_t index = PickIndex(); - GPR_ASSERT(index < subchannels_.size()); - auto& subchannel_info = subchannels_[index]; - // Collect per-call utilization data if needed. - std::unique_ptr subchannel_call_tracker; - if (!config_->enable_oob_load_report()) { - subchannel_call_tracker = std::make_unique( - subchannel_info.weight, config_->error_utilization_penalty()); - } - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_wrr_trace)) { - gpr_log(GPR_INFO, - "[WRR %p picker %p] returning index %" PRIuPTR ", subchannel=%p", - wrr_.get(), this, index, subchannel_info.subchannel.get()); - } - return PickResult::Complete(subchannel_info.subchannel, - std::move(subchannel_call_tracker)); -} - -size_t OldWeightedRoundRobin::Picker::PickIndex() { - // Grab a ref to the scheduler. - std::shared_ptr scheduler; - { - MutexLock lock(&scheduler_mu_); - scheduler = scheduler_; - } - // If we have a scheduler, use it to do a WRR pick. - if (scheduler != nullptr) return scheduler->Pick(); - // We don't have a scheduler (i.e., either all of the weights are 0 or - // there is only one subchannel), so fall back to RR. - return last_picked_index_.fetch_add(1) % subchannels_.size(); -} - -void OldWeightedRoundRobin::Picker::BuildSchedulerAndStartTimerLocked() { - // Build scheduler. - const Timestamp now = Timestamp::Now(); - std::vector weights; - weights.reserve(subchannels_.size()); - for (const auto& subchannel : subchannels_) { - weights.push_back(subchannel.weight->GetWeight( - now, config_->weight_expiration_period(), config_->blackout_period())); - } - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_wrr_trace)) { - gpr_log(GPR_INFO, "[WRR %p picker %p] new weights: %s", wrr_.get(), this, - absl::StrJoin(weights, " ").c_str()); - } - auto scheduler_or = StaticStrideScheduler::Make( - weights, [this]() { return wrr_->scheduler_state_.fetch_add(1); }); - std::shared_ptr scheduler; - if (scheduler_or.has_value()) { - scheduler = - std::make_shared(std::move(*scheduler_or)); - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_wrr_trace)) { - gpr_log(GPR_INFO, "[WRR %p picker %p] new scheduler: %p", wrr_.get(), - this, scheduler.get()); - } - } else if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_wrr_trace)) { - gpr_log(GPR_INFO, "[WRR %p picker %p] no scheduler, falling back to RR", - wrr_.get(), this); - } - { - MutexLock lock(&scheduler_mu_); - scheduler_ = std::move(scheduler); - } - // Start timer. - timer_handle_ = wrr_->channel_control_helper()->GetEventEngine()->RunAfter( - config_->weight_update_period(), - [self = WeakRefAsSubclass(), - work_serializer = wrr_->work_serializer()]() mutable { - ApplicationCallbackExecCtx callback_exec_ctx; - ExecCtx exec_ctx; - { - MutexLock lock(&self->timer_mu_); - if (self->timer_handle_.has_value()) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_wrr_trace)) { - gpr_log(GPR_INFO, "[WRR %p picker %p] timer fired", - self->wrr_.get(), self.get()); - } - self->BuildSchedulerAndStartTimerLocked(); - } - } - if (!IsWorkSerializerDispatchEnabled()) { - // Release the picker ref inside the WorkSerializer. - work_serializer->Run([self = std::move(self)]() {}, DEBUG_LOCATION); - return; - } - self.reset(); - }); -} +#include "src/core/lib/json/json_object_loader.h" +#include "src/core/lib/transport/connectivity_state.h" +#include "src/core/load_balancing/backend_metric_data.h" +#include "src/core/load_balancing/endpoint_list.h" +#include "src/core/load_balancing/oob_backend_metric.h" +#include "src/core/load_balancing/weighted_round_robin/static_stride_scheduler.h" +#include "src/core/load_balancing/weighted_target/weighted_target.h" +#include "src/core/load_balancing/lb_policy.h" +#include "src/core/load_balancing/lb_policy_factory.h" +#include "src/core/load_balancing/subchannel_interface.h" +#include "src/core/resolver/endpoint_addresses.h" -// -// WeightedRoundRobin -// +namespace grpc_core { -OldWeightedRoundRobin::OldWeightedRoundRobin(Args args) - : LoadBalancingPolicy(std::move(args)) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_wrr_trace)) { - gpr_log(GPR_INFO, "[WRR %p] Created", this); - } -} +TraceFlag grpc_lb_wrr_trace(false, "weighted_round_robin_lb"); -OldWeightedRoundRobin::~OldWeightedRoundRobin() { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_wrr_trace)) { - gpr_log(GPR_INFO, "[WRR %p] Destroying Round Robin policy", this); - } - GPR_ASSERT(subchannel_list_ == nullptr); - GPR_ASSERT(latest_pending_subchannel_list_ == nullptr); -} +namespace { -void OldWeightedRoundRobin::ShutdownLocked() { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_wrr_trace)) { - gpr_log(GPR_INFO, "[WRR %p] Shutting down", this); - } - shutdown_ = true; - subchannel_list_.reset(); - latest_pending_subchannel_list_.reset(); -} +constexpr absl::string_view kWeightedRoundRobin = "weighted_round_robin"; -void OldWeightedRoundRobin::ResetBackoffLocked() { - subchannel_list_->ResetBackoffLocked(); - if (latest_pending_subchannel_list_ != nullptr) { - latest_pending_subchannel_list_->ResetBackoffLocked(); - } -} +constexpr absl::string_view kMetricLabelLocality = "grpc.lb.locality"; -absl::Status OldWeightedRoundRobin::UpdateLocked(UpdateArgs args) { - global_stats().IncrementWrrUpdates(); - config_ = args.config.TakeAsSubclass(); - std::shared_ptr addresses; - if (args.addresses.ok()) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_wrr_trace)) { - gpr_log(GPR_INFO, "[WRR %p] received update", this); - } - // Weed out duplicate addresses. Also sort the addresses so that if - // the set of the addresses don't change, their indexes in the - // subchannel list don't change, since this avoids unnecessary churn - // in the picker. Note that this does not ensure that if a given - // address remains present that it will have the same index; if, - // for example, an address at the end of the list is replaced with one - // that sorts much earlier in the list, then all of the addresses in - // between those two positions will have changed indexes. - struct AddressLessThan { - bool operator()(const ServerAddress& address1, - const ServerAddress& address2) const { - const grpc_resolved_address& addr1 = address1.address(); - const grpc_resolved_address& addr2 = address2.address(); - if (addr1.len != addr2.len) return addr1.len < addr2.len; - return memcmp(addr1.addr, addr2.addr, addr1.len) < 0; - } - }; - std::set ordered_addresses; - (*args.addresses)->ForEach([&](const EndpointAddresses& endpoint) { - ordered_addresses.insert(endpoint); - }); - addresses = std::make_shared( - ServerAddressList(ordered_addresses.begin(), ordered_addresses.end())); - } else { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_wrr_trace)) { - gpr_log(GPR_INFO, "[WRR %p] received update with address error: %s", this, - args.addresses.status().ToString().c_str()); - } - // If we already have a subchannel list, then keep using the existing - // list, but still report back that the update was not accepted. - if (subchannel_list_ != nullptr) return args.addresses.status(); - } - // Create new subchannel list, replacing the previous pending list, if any. - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_wrr_trace) && - latest_pending_subchannel_list_ != nullptr) { - gpr_log(GPR_INFO, "[WRR %p] replacing previous pending subchannel list %p", - this, latest_pending_subchannel_list_.get()); - } - latest_pending_subchannel_list_ = - MakeRefCounted(this, addresses.get(), - args.args); - latest_pending_subchannel_list_->StartWatchingLocked(args.args); - // If the new list is empty, immediately promote it to - // subchannel_list_ and report TRANSIENT_FAILURE. - if (latest_pending_subchannel_list_->num_subchannels() == 0) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_wrr_trace) && - subchannel_list_ != nullptr) { - gpr_log(GPR_INFO, "[WRR %p] replacing previous subchannel list %p", this, - subchannel_list_.get()); - } - subchannel_list_ = std::move(latest_pending_subchannel_list_); - absl::Status status = - args.addresses.ok() ? absl::UnavailableError(absl::StrCat( - "empty address list: ", args.resolution_note)) - : args.addresses.status(); - channel_control_helper()->UpdateState( - GRPC_CHANNEL_TRANSIENT_FAILURE, status, - MakeRefCounted(status)); - return status; - } - // Otherwise, if this is the initial update, immediately promote it to - // subchannel_list_. - if (subchannel_list_.get() == nullptr) { - subchannel_list_ = std::move(latest_pending_subchannel_list_); - } - return absl::OkStatus(); -} +const auto kMetricRrFallback = + GlobalInstrumentsRegistry::RegisterUInt64Counter( + "grpc.lb.wrr.rr_fallback", + "EXPERIMENTAL. Number of scheduler updates in which there were not " + "enough endpoints with valid weight, which caused the WRR policy to " + "fall back to RR behavior.", + "{update}", {kMetricLabelTarget}, {kMetricLabelLocality}, false); -RefCountedPtr -OldWeightedRoundRobin::GetOrCreateWeight(const grpc_resolved_address& address) { - auto key = grpc_sockaddr_to_uri(&address); - if (!key.ok()) return nullptr; - MutexLock lock(&address_weight_map_mu_); - auto it = address_weight_map_.find(*key); - if (it != address_weight_map_.end()) { - auto weight = it->second->RefIfNonZero(); - if (weight != nullptr) return weight; - } - auto weight = MakeRefCounted( - RefAsSubclass(DEBUG_LOCATION, "AddressWeight"), - *key); - address_weight_map_.emplace(*key, weight.get()); - return weight; -} +const auto kMetricEndpointWeightNotYetUsable = + GlobalInstrumentsRegistry::RegisterUInt64Counter( + "grpc.lb.wrr.endpoint_weight_not_yet_usable", + "EXPERIMENTAL. Number of endpoints from each scheduler update that " + "don't yet have usable weight information (i.e., either the load " + "report has not yet been received, or it is within the blackout " + "period).", + "{endpoint}", {kMetricLabelTarget}, {kMetricLabelLocality}, false); -// -// OldWeightedRoundRobin::WeightedRoundRobinSubchannelList -// +const auto kMetricEndpointWeightStale = + GlobalInstrumentsRegistry::RegisterUInt64Counter( + "grpc.lb.wrr.endpoint_weight_stale", + "EXPERIMENTAL. Number of endpoints from each scheduler update whose " + "latest weight is older than the expiration period.", + "{endpoint}", {kMetricLabelTarget}, {kMetricLabelLocality}, false); -void OldWeightedRoundRobin::WeightedRoundRobinSubchannelList:: - UpdateStateCountersLocked(absl::optional old_state, - grpc_connectivity_state new_state) { - if (old_state.has_value()) { - GPR_ASSERT(*old_state != GRPC_CHANNEL_SHUTDOWN); - if (*old_state == GRPC_CHANNEL_READY) { - GPR_ASSERT(num_ready_ > 0); - --num_ready_; - } else if (*old_state == GRPC_CHANNEL_CONNECTING) { - GPR_ASSERT(num_connecting_ > 0); - --num_connecting_; - } else if (*old_state == GRPC_CHANNEL_TRANSIENT_FAILURE) { - GPR_ASSERT(num_transient_failure_ > 0); - --num_transient_failure_; - } - } - GPR_ASSERT(new_state != GRPC_CHANNEL_SHUTDOWN); - if (new_state == GRPC_CHANNEL_READY) { - ++num_ready_; - } else if (new_state == GRPC_CHANNEL_CONNECTING) { - ++num_connecting_; - } else if (new_state == GRPC_CHANNEL_TRANSIENT_FAILURE) { - ++num_transient_failure_; - } -} +const auto kMetricEndpointWeights = + GlobalInstrumentsRegistry::RegisterDoubleHistogram( + "grpc.lb.wrr.endpoint_weights", + "EXPERIMENTAL. The histogram buckets will be endpoint weight ranges. " + "Each bucket will be a counter that is incremented once for every " + "endpoint whose weight is within that range. Note that endpoints " + "without usable weights will have weight 0.", + "{weight}", {kMetricLabelTarget}, {kMetricLabelLocality}, false); -void OldWeightedRoundRobin::WeightedRoundRobinSubchannelList:: - MaybeUpdateAggregatedConnectivityStateLocked(absl::Status status_for_tf) { - OldWeightedRoundRobin* p = static_cast(policy()); - // If this is latest_pending_subchannel_list_, then swap it into - // subchannel_list_ in the following cases: - // - subchannel_list_ has no READY subchannels. - // - This list has at least one READY subchannel and we have seen the - // initial connectivity state notification for all subchannels. - // - All of the subchannels in this list are in TRANSIENT_FAILURE. - // (This may cause the channel to go from READY to TRANSIENT_FAILURE, - // but we're doing what the control plane told us to do.) - if (p->latest_pending_subchannel_list_.get() == this && - (p->subchannel_list_->num_ready_ == 0 || - (num_ready_ > 0 && AllSubchannelsSeenInitialState()) || - num_transient_failure_ == num_subchannels())) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_wrr_trace)) { - const std::string old_counters_string = - p->subchannel_list_ != nullptr ? p->subchannel_list_->CountersString() - : ""; - gpr_log( - GPR_INFO, - "[WRR %p] swapping out subchannel list %p (%s) in favor of %p (%s)", - p, p->subchannel_list_.get(), old_counters_string.c_str(), this, - CountersString().c_str()); - } - p->subchannel_list_ = std::move(p->latest_pending_subchannel_list_); - } - // Only set connectivity state if this is the current subchannel list. - if (p->subchannel_list_.get() != this) return; - // First matching rule wins: - // 1) ANY subchannel is READY => policy is READY. - // 2) ANY subchannel is CONNECTING => policy is CONNECTING. - // 3) ALL subchannels are TRANSIENT_FAILURE => policy is TRANSIENT_FAILURE. - if (num_ready_ > 0) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_wrr_trace)) { - gpr_log(GPR_INFO, "[WRR %p] reporting READY with subchannel list %p", p, - this); - } - p->channel_control_helper()->UpdateState( - GRPC_CHANNEL_READY, absl::Status(), - MakeRefCounted(p->RefAsSubclass(), - this)); - } else if (num_connecting_ > 0) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_wrr_trace)) { - gpr_log(GPR_INFO, "[WRR %p] reporting CONNECTING with subchannel list %p", - p, this); - } - p->channel_control_helper()->UpdateState( - GRPC_CHANNEL_CONNECTING, absl::Status(), - MakeRefCounted(p->Ref(DEBUG_LOCATION, "QueuePicker"))); - } else if (num_transient_failure_ == num_subchannels()) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_wrr_trace)) { - gpr_log( - GPR_INFO, - "[WRR %p] reporting TRANSIENT_FAILURE with subchannel list %p: %s", p, - this, status_for_tf.ToString().c_str()); - } - if (!status_for_tf.ok()) { - last_failure_ = absl::UnavailableError( - absl::StrCat("connections to all backends failing; last error: ", - status_for_tf.ToString())); - } - p->channel_control_helper()->UpdateState( - GRPC_CHANNEL_TRANSIENT_FAILURE, last_failure_, - MakeRefCounted(last_failure_)); - } -} +// Config for WRR policy. +class WeightedRoundRobinConfig : public LoadBalancingPolicy::Config { + public: + WeightedRoundRobinConfig() = default; -// -// OldWeightedRoundRobin::WeightedRoundRobinSubchannelData::OobWatcher -// + WeightedRoundRobinConfig(const WeightedRoundRobinConfig&) = delete; + WeightedRoundRobinConfig& operator=(const WeightedRoundRobinConfig&) = delete; -void OldWeightedRoundRobin::WeightedRoundRobinSubchannelData::OobWatcher:: - OnBackendMetricReport(const BackendMetricData& backend_metric_data) { - double utilization = backend_metric_data.application_utilization; - if (utilization <= 0) { - utilization = backend_metric_data.cpu_utilization; - } - weight_->MaybeUpdateWeight(backend_metric_data.qps, backend_metric_data.eps, - utilization, error_utilization_penalty_); -} + WeightedRoundRobinConfig(WeightedRoundRobinConfig&&) = delete; + WeightedRoundRobinConfig& operator=(WeightedRoundRobinConfig&&) = delete; -// -// OldWeightedRoundRobin::WeightedRoundRobinSubchannelData -// + absl::string_view name() const override { return kWeightedRoundRobin; } -OldWeightedRoundRobin::WeightedRoundRobinSubchannelData:: - WeightedRoundRobinSubchannelData( - SubchannelList* subchannel_list, - const ServerAddress& address, RefCountedPtr sc) - : SubchannelData(subchannel_list, address, std::move(sc)), - weight_(static_cast(subchannel_list->policy()) - ->GetOrCreateWeight(address.address())) { - // Start OOB watch if configured. - OldWeightedRoundRobin* p = - static_cast(subchannel_list->policy()); - if (p->config_->enable_oob_load_report()) { - subchannel()->AddDataWatcher(MakeOobBackendMetricWatcher( - p->config_->oob_reporting_period(), - std::make_unique(weight_, - p->config_->error_utilization_penalty()))); + bool enable_oob_load_report() const { return enable_oob_load_report_; } + Duration oob_reporting_period() const { return oob_reporting_period_; } + Duration blackout_period() const { return blackout_period_; } + Duration weight_update_period() const { return weight_update_period_; } + Duration weight_expiration_period() const { + return weight_expiration_period_; } -} + float error_utilization_penalty() const { return error_utilization_penalty_; } -void OldWeightedRoundRobin::WeightedRoundRobinSubchannelData:: - ProcessConnectivityChangeLocked( - absl::optional old_state, - grpc_connectivity_state new_state) { - OldWeightedRoundRobin* p = - static_cast(subchannel_list()->policy()); - GPR_ASSERT(subchannel() != nullptr); - // If this is not the initial state notification and the new state is - // TRANSIENT_FAILURE or IDLE, re-resolve. - // Note that we don't want to do this on the initial state notification, - // because that would result in an endless loop of re-resolution. - if (old_state.has_value() && (new_state == GRPC_CHANNEL_TRANSIENT_FAILURE || - new_state == GRPC_CHANNEL_IDLE)) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_wrr_trace)) { - gpr_log(GPR_INFO, - "[WRR %p] Subchannel %p reported %s; requesting re-resolution", p, - subchannel(), ConnectivityStateName(new_state)); - } - p->channel_control_helper()->RequestReresolution(); - } - if (new_state == GRPC_CHANNEL_IDLE) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_wrr_trace)) { - gpr_log(GPR_INFO, - "[WRR %p] Subchannel %p reported IDLE; requesting connection", p, - subchannel()); - } - subchannel()->RequestConnection(); - } else if (new_state == GRPC_CHANNEL_READY) { - // If we transition back to READY state, restart the blackout period. - // Skip this if this is the initial notification for this - // subchannel (which happens whenever we get updated addresses and - // create a new endpoint list). Also skip it if the previous state - // was READY (which should never happen in practice, but we've seen - // at least one bug that caused this in the outlier_detection - // policy, so let's be defensive here). - // - // Note that we cannot guarantee that we will never receive - // lingering callbacks for backend metric reports from the previous - // connection after the new connection has been established, but they - // should be masked by new backend metric reports from the new - // connection by the time the blackout period ends. - if (old_state.has_value() && old_state != GRPC_CHANNEL_READY) { - weight_->ResetNonEmptySince(); - } + static const JsonLoaderInterface* JsonLoader(const JsonArgs&) { + static const auto* loader = + JsonObjectLoader() + .OptionalField("enableOobLoadReport", + &WeightedRoundRobinConfig::enable_oob_load_report_) + .OptionalField("oobReportingPeriod", + &WeightedRoundRobinConfig::oob_reporting_period_) + .OptionalField("blackoutPeriod", + &WeightedRoundRobinConfig::blackout_period_) + .OptionalField("weightUpdatePeriod", + &WeightedRoundRobinConfig::weight_update_period_) + .OptionalField("weightExpirationPeriod", + &WeightedRoundRobinConfig::weight_expiration_period_) + .OptionalField( + "errorUtilizationPenalty", + &WeightedRoundRobinConfig::error_utilization_penalty_) + .Finish(); + return loader; } - // Update logical connectivity state. - UpdateLogicalConnectivityStateLocked(new_state); - // Update the policy state. - subchannel_list()->MaybeUpdateAggregatedConnectivityStateLocked( - connectivity_status()); -} -void OldWeightedRoundRobin::WeightedRoundRobinSubchannelData:: - UpdateLogicalConnectivityStateLocked( - grpc_connectivity_state connectivity_state) { - OldWeightedRoundRobin* p = - static_cast(subchannel_list()->policy()); - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_wrr_trace)) { - gpr_log( - GPR_INFO, - "[WRR %p] connectivity changed for subchannel %p, subchannel_list %p " - "(index %" PRIuPTR " of %" PRIuPTR "): prev_state=%s new_state=%s", - p, subchannel(), subchannel_list(), Index(), - subchannel_list()->num_subchannels(), - (logical_connectivity_state_.has_value() - ? ConnectivityStateName(*logical_connectivity_state_) - : "N/A"), - ConnectivityStateName(connectivity_state)); - } - // Decide what state to report for aggregation purposes. - // If the last logical state was TRANSIENT_FAILURE, then ignore the - // state change unless the new state is READY. - if (logical_connectivity_state_.has_value() && - *logical_connectivity_state_ == GRPC_CHANNEL_TRANSIENT_FAILURE && - connectivity_state != GRPC_CHANNEL_READY) { - return; - } - // If the new state is IDLE, treat it as CONNECTING, since it will - // immediately transition into CONNECTING anyway. - if (connectivity_state == GRPC_CHANNEL_IDLE) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_wrr_trace)) { - gpr_log(GPR_INFO, - "[WRR %p] subchannel %p, subchannel_list %p (index %" PRIuPTR - " of %" PRIuPTR "): treating IDLE as CONNECTING", - p, subchannel(), subchannel_list(), Index(), - subchannel_list()->num_subchannels()); + void JsonPostLoad(const Json&, const JsonArgs&, ValidationErrors* errors) { + // Impose lower bound of 100ms on weightUpdatePeriod. + weight_update_period_ = + std::max(weight_update_period_, Duration::Milliseconds(100)); + if (error_utilization_penalty_ < 0) { + ValidationErrors::ScopedField field(errors, ".errorUtilizationPenalty"); + errors->AddError("must be non-negative"); } - connectivity_state = GRPC_CHANNEL_CONNECTING; - } - // If no change, return false. - if (logical_connectivity_state_.has_value() && - *logical_connectivity_state_ == connectivity_state) { - return; } - // Otherwise, update counters and logical state. - subchannel_list()->UpdateStateCountersLocked(logical_connectivity_state_, - connectivity_state); - logical_connectivity_state_ = connectivity_state; -} -// New WRR LB policy (with delegation to pick_first) + private: + bool enable_oob_load_report_ = false; + Duration oob_reporting_period_ = Duration::Seconds(10); + Duration blackout_period_ = Duration::Seconds(10); + Duration weight_update_period_ = Duration::Seconds(1); + Duration weight_expiration_period_ = Duration::Minutes(3); + float error_utilization_penalty_ = 1.0; +}; + +// WRR LB policy class WeightedRoundRobin : public LoadBalancingPolicy { public: explicit WeightedRoundRobin(Args args); @@ -1858,9 +1002,6 @@ class WeightedRoundRobinFactory : public LoadBalancingPolicyFactory { public: OrphanablePtr CreateLoadBalancingPolicy( LoadBalancingPolicy::Args args) const override { - if (!IsWrrDelegateToPickFirstEnabled()) { - return MakeOrphanable(std::move(args)); - } return MakeOrphanable(std::move(args)); } diff --git a/test/core/client_channel/lb_policy/outlier_detection_test.cc b/test/core/client_channel/lb_policy/outlier_detection_test.cc index 8e93de35f0f..1dbb5476037 100644 --- a/test/core/client_channel/lb_policy/outlier_detection_test.cc +++ b/test/core/client_channel/lb_policy/outlier_detection_test.cc @@ -35,7 +35,6 @@ #include #include -#include "src/core/lib/experiments/experiments.h" #include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/gprpp/time.h" @@ -224,7 +223,6 @@ TEST_F(OutlierDetectionTest, FailurePercentage) { // Advance time and run the timer callback to trigger ejection. IncrementTimeBy(Duration::Seconds(10)); gpr_log(GPR_INFO, "### ejection complete"); - if (!IsRoundRobinDelegateToPickFirstEnabled()) ExpectReresolutionRequest(); // Expect a picker update. std::vector remaining_addresses; for (const auto& addr : kAddresses) { @@ -239,7 +237,6 @@ TEST_F(OutlierDetectionTest, FailurePercentage) { } TEST_F(OutlierDetectionTest, MultipleAddressesPerEndpoint) { - if (!IsRoundRobinDelegateToPickFirstEnabled()) return; // Can't use timer duration expectation here, because the Happy // Eyeballs timer inside pick_first will use a different duration than // the timer in outlier_detection. @@ -338,7 +335,6 @@ TEST_F(OutlierDetectionTest, MultipleAddressesPerEndpoint) { } TEST_F(OutlierDetectionTest, EjectionStateResetsWhenEndpointAddressesChange) { - if (!IsRoundRobinDelegateToPickFirstEnabled()) return; // Can't use timer duration expectation here, because the Happy // Eyeballs timer inside pick_first will use a different duration than // the timer in outlier_detection. diff --git a/test/core/client_channel/lb_policy/round_robin_test.cc b/test/core/client_channel/lb_policy/round_robin_test.cc index ce49f7ef6b4..9af35ed951d 100644 --- a/test/core/client_channel/lb_policy/round_robin_test.cc +++ b/test/core/client_channel/lb_policy/round_robin_test.cc @@ -24,7 +24,6 @@ #include -#include "src/core/lib/experiments/experiments.h" #include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/resolver/endpoint_addresses.h" @@ -70,7 +69,6 @@ TEST_F(RoundRobinTest, AddressUpdates) { } TEST_F(RoundRobinTest, MultipleAddressesPerEndpoint) { - if (!IsRoundRobinDelegateToPickFirstEnabled()) return; constexpr std::array kEndpoint1Addresses = { "ipv4:127.0.0.1:443", "ipv4:127.0.0.1:444"}; constexpr std::array kEndpoint2Addresses = { diff --git a/test/core/client_channel/lb_policy/weighted_round_robin_test.cc b/test/core/client_channel/lb_policy/weighted_round_robin_test.cc index 4572429513e..edc89309eca 100644 --- a/test/core/client_channel/lb_policy/weighted_round_robin_test.cc +++ b/test/core/client_channel/lb_policy/weighted_round_robin_test.cc @@ -40,7 +40,6 @@ #include #include -#include "src/core/lib/experiments/experiments.h" #include "src/core/lib/gprpp/debug_location.h" #include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" @@ -858,7 +857,6 @@ TEST_F(WeightedRoundRobinTest, ZeroErrorUtilPenalty) { } TEST_F(WeightedRoundRobinTest, MultipleAddressesPerEndpoint) { - if (!IsWrrDelegateToPickFirstEnabled()) return; // Can't use timer duration expectation here, because the Happy // Eyeballs timer inside pick_first will use a different duration than // the timer in WRR. @@ -1066,7 +1064,6 @@ TEST_F(WeightedRoundRobinTest, MetricDefinitionEndpointWeights) { } TEST_F(WeightedRoundRobinTest, MetricValues) { - if (!IsWrrDelegateToPickFirstEnabled()) return; const auto kRrFallback = GlobalInstrumentsRegistryTestPeer::FindUInt64CounterHandleByName( "grpc.lb.wrr.rr_fallback") diff --git a/test/core/client_channel/lb_policy/xds_override_host_test.cc b/test/core/client_channel/lb_policy/xds_override_host_test.cc index d1ddeae05c7..e6c74b7b2f1 100644 --- a/test/core/client_channel/lb_policy/xds_override_host_test.cc +++ b/test/core/client_channel/lb_policy/xds_override_host_test.cc @@ -40,7 +40,6 @@ #include "src/core/ext/filters/stateful_session/stateful_session_filter.h" #include "src/core/ext/xds/xds_health_status.h" #include "src/core/lib/channel/channel_args.h" -#include "src/core/lib/experiments/experiments.h" #include "src/core/lib/gprpp/debug_location.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/json/json.h" @@ -511,7 +510,6 @@ TEST_F(XdsOverrideHostTest, OverrideHostStatus) { } TEST_F(XdsOverrideHostTest, MultipleAddressesPerEndpoint) { - if (!IsRoundRobinDelegateToPickFirstEnabled()) return; constexpr std::array kEndpoint1Addresses = { "ipv4:127.0.0.1:443", "ipv4:127.0.0.1:444"}; constexpr std::array kEndpoint2Addresses = { diff --git a/test/cpp/end2end/client_lb_end2end_test.cc b/test/cpp/end2end/client_lb_end2end_test.cc index 11bc2492cdd..d5ee65ad9d0 100644 --- a/test/cpp/end2end/client_lb_end2end_test.cc +++ b/test/cpp/end2end/client_lb_end2end_test.cc @@ -56,7 +56,6 @@ #include "src/core/lib/backoff/backoff.h" #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/config/config_vars.h" -#include "src/core/lib/experiments/experiments.h" #include "src/core/lib/gprpp/crash.h" #include "src/core/lib/gprpp/debug_location.h" #include "src/core/lib/gprpp/env.h" @@ -1982,13 +1981,8 @@ TEST_F(RoundRobinTest, HealthChecking) { EXPECT_THAT( status.error_message(), ::testing::MatchesRegex( - grpc_core::IsRoundRobinDelegateToPickFirstEnabled() - ? "connections to all backends failing; last error: " - "(ipv6:%5B::1%5D|ipv4:127.0.0.1):[0-9]+: " - "backend unhealthy" - : "connections to all backends failing; last error: " - "UNAVAILABLE: (ipv6:%5B::1%5D|ipv4:127.0.0.1):[0-9]+: " - "backend unhealthy")); + "connections to all backends failing; last error: " + "(ipv6:%5B::1%5D|ipv4:127.0.0.1):[0-9]+: backend unhealthy")); return false; }); // Clean up. @@ -2048,13 +2042,8 @@ TEST_F(RoundRobinTest, WithHealthCheckingInhibitPerChannel) { EXPECT_FALSE(WaitForChannelReady(channel1.get(), 1)); CheckRpcSendFailure( DEBUG_LOCATION, stub1, StatusCode::UNAVAILABLE, - grpc_core::IsRoundRobinDelegateToPickFirstEnabled() - ? "connections to all backends failing; last error: " - "(ipv6:%5B::1%5D|ipv4:127.0.0.1):[0-9]+: " - "backend unhealthy" - : "connections to all backends failing; last error: " - "UNAVAILABLE: (ipv6:%5B::1%5D|ipv4:127.0.0.1):[0-9]+: " - "backend unhealthy"); + "connections to all backends failing; last error: " + "(ipv6:%5B::1%5D|ipv4:127.0.0.1):[0-9]+: backend unhealthy"); // Second channel should be READY. EXPECT_TRUE(WaitForChannelReady(channel2.get(), 1)); CheckRpcSendOk(DEBUG_LOCATION, stub2); @@ -2099,13 +2088,8 @@ TEST_F(RoundRobinTest, HealthCheckingServiceNamePerChannel) { EXPECT_FALSE(WaitForChannelReady(channel1.get(), 1)); CheckRpcSendFailure( DEBUG_LOCATION, stub1, StatusCode::UNAVAILABLE, - grpc_core::IsRoundRobinDelegateToPickFirstEnabled() - ? "connections to all backends failing; last error: " - "(ipv6:%5B::1%5D|ipv4:127.0.0.1):[0-9]+: " - "backend unhealthy" - : "connections to all backends failing; last error: " - "UNAVAILABLE: (ipv6:%5B::1%5D|ipv4:127.0.0.1):[0-9]+: " - "backend unhealthy"); + "connections to all backends failing; last error: " + "(ipv6:%5B::1%5D|ipv4:127.0.0.1):[0-9]+: backend unhealthy"); // Second channel should be READY. EXPECT_TRUE(WaitForChannelReady(channel2.get(), 1)); CheckRpcSendOk(DEBUG_LOCATION, stub2); diff --git a/test/cpp/end2end/xds/xds_cluster_end2end_test.cc b/test/cpp/end2end/xds/xds_cluster_end2end_test.cc index 7322f06c9f6..2772a2370cf 100644 --- a/test/cpp/end2end/xds/xds_cluster_end2end_test.cc +++ b/test/cpp/end2end/xds/xds_cluster_end2end_test.cc @@ -27,7 +27,6 @@ #include "src/core/lib/address_utils/sockaddr_utils.h" #include "src/core/lib/channel/call_tracer.h" #include "src/core/lib/config/config_vars.h" -#include "src/core/lib/experiments/experiments.h" #include "src/core/lib/surface/call.h" #include "src/proto/grpc/testing/xds/v3/orca_load_report.pb.h" #include "test/core/util/fake_stats_plugin.h" @@ -482,7 +481,6 @@ TEST_P(EdsTest, Vanilla) { } TEST_P(EdsTest, MultipleAddressesPerEndpoint) { - if (!grpc_core::IsRoundRobinDelegateToPickFirstEnabled()) return; grpc_core::testing::ScopedExperimentalEnvVar env( "GRPC_EXPERIMENTAL_XDS_DUALSTACK_ENDPOINTS"); const size_t kNumRpcsPerAddress = 10; diff --git a/test/cpp/end2end/xds/xds_override_host_end2end_test.cc b/test/cpp/end2end/xds/xds_override_host_end2end_test.cc index 65d5c43f9bb..fbd3793b206 100644 --- a/test/cpp/end2end/xds/xds_override_host_end2end_test.cc +++ b/test/cpp/end2end/xds/xds_override_host_end2end_test.cc @@ -23,7 +23,6 @@ #include "absl/strings/str_split.h" #include "src/core/lib/config/config_vars.h" -#include "src/core/lib/experiments/experiments.h" #include "src/core/lib/gprpp/time.h" #include "src/proto/grpc/testing/xds/v3/stateful_session.pb.h" #include "src/proto/grpc/testing/xds/v3/stateful_session_cookie.pb.h" @@ -703,7 +702,6 @@ TEST_P(OverrideHostTest, TTLSetsMaxAge) { } TEST_P(OverrideHostTest, MultipleAddressesPerEndpoint) { - if (!grpc_core::IsRoundRobinDelegateToPickFirstEnabled()) return; grpc_core::testing::ScopedExperimentalEnvVar env( "GRPC_EXPERIMENTAL_XDS_DUALSTACK_ENDPOINTS"); // Create 3 backends, but leave backend 0 unstarted. diff --git a/test/cpp/end2end/xds/xds_wrr_end2end_test.cc b/test/cpp/end2end/xds/xds_wrr_end2end_test.cc index a89a3021e73..88f7513b532 100644 --- a/test/cpp/end2end/xds/xds_wrr_end2end_test.cc +++ b/test/cpp/end2end/xds/xds_wrr_end2end_test.cc @@ -27,7 +27,6 @@ #include "src/core/client_channel/backup_poller.h" #include "src/core/lib/config/config_vars.h" -#include "src/core/lib/experiments/experiments.h" #include "src/proto/grpc/testing/xds/v3/client_side_weighted_round_robin.grpc.pb.h" #include "src/proto/grpc/testing/xds/v3/wrr_locality.grpc.pb.h" #include "test/core/util/fake_stats_plugin.h" @@ -102,7 +101,6 @@ TEST_P(WrrTest, Basic) { } TEST_P(WrrTest, MetricsHaveLocalityLabel) { - if (!grpc_core::IsWrrDelegateToPickFirstEnabled()) return; const auto kEndpointWeights = grpc_core::GlobalInstrumentsRegistryTestPeer:: FindDoubleHistogramHandleByName("grpc.lb.wrr.endpoint_weights") diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 35da52b9935..ba4cbaec282 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -2897,7 +2897,6 @@ src/core/load_balancing/rls/rls.cc \ src/core/load_balancing/rls/rls.h \ src/core/load_balancing/round_robin/round_robin.cc \ src/core/load_balancing/subchannel_interface.h \ -src/core/load_balancing/subchannel_list.h \ src/core/load_balancing/weighted_round_robin/static_stride_scheduler.cc \ src/core/load_balancing/weighted_round_robin/static_stride_scheduler.h \ src/core/load_balancing/weighted_round_robin/weighted_round_robin.cc \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 1f4c698de8b..8796adfd4c6 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -2674,7 +2674,6 @@ src/core/load_balancing/rls/rls.cc \ src/core/load_balancing/rls/rls.h \ src/core/load_balancing/round_robin/round_robin.cc \ src/core/load_balancing/subchannel_interface.h \ -src/core/load_balancing/subchannel_list.h \ src/core/load_balancing/weighted_round_robin/static_stride_scheduler.cc \ src/core/load_balancing/weighted_round_robin/static_stride_scheduler.h \ src/core/load_balancing/weighted_round_robin/weighted_round_robin.cc \ From 88585c43e902e2088928c55ba738a91c0cf48835 Mon Sep 17 00:00:00 2001 From: Yijie Ma Date: Tue, 12 Mar 2024 23:27:08 -0700 Subject: [PATCH 37/52] [test] Allow channel-level "Sending goaway" log in no_logging test (#36100) Closes #36100 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/36100 from yijiem:no-logging 970a6404759f7e0e0e667a21e2469b55e45c4d17 PiperOrigin-RevId: 615299752 --- test/core/end2end/tests/no_logging.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/core/end2end/tests/no_logging.cc b/test/core/end2end/tests/no_logging.cc index 774d7bfe712..d83537529ae 100644 --- a/test/core/end2end/tests/no_logging.cc +++ b/test/core/end2end/tests/no_logging.cc @@ -73,7 +73,9 @@ class Verifier { static void NoLog(gpr_log_func_args* args) { static const auto* const allowed_logs_by_module = new std::map( - {{"cq_verifier.cc", std::regex("^Verify .* for [0-9]+ms")}}); + {{"cq_verifier.cc", std::regex("^Verify .* for [0-9]+ms")}, + {"chttp2_transport.cc", + std::regex("Sending goaway.*Channel Destroyed")}}); absl::string_view filename = args->file; auto slash = filename.rfind('/'); if (slash != absl::string_view::npos) { From 9ed686902e79b113a8d97b3428a0da8adc2dc567 Mon Sep 17 00:00:00 2001 From: Vignesh Babu Date: Wed, 13 Mar 2024 11:09:13 -0700 Subject: [PATCH 38/52] [chttp2-server] Handle error cases correctly when limiting number of accepted connections (#36101) Closes #36101 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/36101 from Vignesh2208:server-fix 41d6992b5c3c1690df7fe1db883adb7f6d3dbd57 PiperOrigin-RevId: 615480491 --- .../transport/chttp2/server/chttp2_server.cc | 7 +++ test/cpp/end2end/BUILD | 2 + .../resource_quota_end2end_stress_test.cc | 58 +++++++++++++++++-- 3 files changed, 63 insertions(+), 4 deletions(-) diff --git a/src/core/ext/transport/chttp2/server/chttp2_server.cc b/src/core/ext/transport/chttp2/server/chttp2_server.cc index 7ac39d01cff..d55590e30e0 100644 --- a/src/core/ext/transport/chttp2/server/chttp2_server.cc +++ b/src/core/ext/transport/chttp2/server/chttp2_server.cc @@ -452,11 +452,13 @@ void Chttp2ServerListener::ActiveConnection::HandshakingState::OnHandshakeDone( OrphanablePtr handshaking_state_ref; RefCountedPtr handshake_mgr; bool cleanup_connection = false; + bool release_connection = false; { MutexLock connection_lock(&self->connection_->mu_); if (!error.ok() || self->connection_->shutdown_) { std::string error_str = StatusToString(error); cleanup_connection = true; + release_connection = true; if (error.ok() && args->endpoint != nullptr) { // We were shut down or stopped serving after handshaking completed // successfully, so destroy the endpoint here. @@ -540,9 +542,11 @@ void Chttp2ServerListener::ActiveConnection::HandshakingState::OnHandshakeDone( grpc_slice_buffer_destroy(args->read_buffer); gpr_free(args->read_buffer); cleanup_connection = true; + release_connection = true; } } else { cleanup_connection = true; + release_connection = true; } } // Since the handshake manager is done, the connection no longer needs to @@ -557,6 +561,9 @@ void Chttp2ServerListener::ActiveConnection::HandshakingState::OnHandshakeDone( OrphanablePtr connection; if (cleanup_connection) { MutexLock listener_lock(&self->connection_->listener_->mu_); + if (release_connection) { + self->connection_->listener_->connection_quota_->ReleaseConnections(1); + } auto it = self->connection_->listener_->connections_.find( self->connection_.get()); if (it != self->connection_->listener_->connections_.end()) { diff --git a/test/cpp/end2end/BUILD b/test/cpp/end2end/BUILD index f0f5305a67d..1a063cfe853 100644 --- a/test/cpp/end2end/BUILD +++ b/test/cpp/end2end/BUILD @@ -1030,7 +1030,9 @@ grpc_cc_test( ], deps = [ "//:grpc++", + "//src/core:event_engine_tcp_socket_utils", "//src/core:experiments", + "//src/core:grpc_fake_credentials", "//src/proto/grpc/testing:echo_messages_proto", "//src/proto/grpc/testing:echo_proto", "//test/core/util:grpc_test_util", diff --git a/test/cpp/end2end/resource_quota_end2end_stress_test.cc b/test/cpp/end2end/resource_quota_end2end_stress_test.cc index 4a8f28deaeb..19c9a9ad6c6 100644 --- a/test/cpp/end2end/resource_quota_end2end_stress_test.cc +++ b/test/cpp/end2end/resource_quota_end2end_stress_test.cc @@ -24,17 +24,24 @@ #include "absl/strings/str_cat.h" #include "absl/time/time.h" +#include #include #include #include #include +#include "src/core/lib/event_engine/tcp_socket_utils.h" #include "src/core/lib/experiments/config.h" #include "src/core/lib/gprpp/notification.h" +#include "src/core/lib/security/credentials/fake/fake_credentials.h" +#include "src/cpp/client/secure_credentials.h" +#include "src/cpp/server/secure_server_credentials.h" #include "src/proto/grpc/testing/echo.grpc.pb.h" #include "test/core/util/port.h" #include "test/core/util/test_config.h" +// IWYU pragma: no_include + // A stress test which spins up a server with a small configured resource quota // value. It then creates many channels which exchange large payloads with the // server. This would drive the server to reach resource quota limits and @@ -150,9 +157,14 @@ class End2EndConnectionQuotaTest : public ::testing::TestWithParam { End2EndConnectionQuotaTest() { port_ = grpc_pick_unused_port_or_die(); server_address_ = absl::StrCat("[::]:", port_); + connect_address_ = absl::StrCat("ipv6:[::1]:", port_); payload_ = std::string(kPayloadSizeBytes, 'a'); ServerBuilder builder; - builder.AddListeningPort(server_address_, InsecureServerCredentials()); + builder.AddListeningPort( + server_address_, + std::make_shared( + grpc_fake_transport_security_server_credentials_create())); + builder.AddChannelArgument(GRPC_ARG_SERVER_HANDSHAKE_TIMEOUT_MS, 1000); builder.AddChannelArgument(GRPC_ARG_MAX_ALLOWED_INCOMING_CONNECTIONS, GetParam()); builder.AddChannelArgument( @@ -172,13 +184,50 @@ class End2EndConnectionQuotaTest : public ::testing::TestWithParam { args.SetInt(GRPC_ARG_HTTP2_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS, 15000); args.SetInt(GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS, 1); - return EchoTestService::NewStub( - CreateCustomChannel(absl::StrCat("ipv6:[::1]:", port_), - grpc::InsecureChannelCredentials(), args)); + return EchoTestService::NewStub(CreateCustomChannel( + connect_address_, + std::make_shared( + grpc_fake_transport_security_credentials_create()), + args)); } void TestExceedingConnectionQuota() { const int kNumConnections = 2 * GetParam(); +#ifdef GPR_LINUX + // On linux systems create 2 * NumConnection tcp connections which don't + // do anything and verify that they get closed after + // GRPC_ARG_SERVER_HANDSHAKE_TIMEOUT_MS seconds. + auto connect_address_resolved = + grpc_event_engine::experimental::URIToResolvedAddress(connect_address_); + std::vector workers; + workers.reserve(kNumConnections); + for (int i = 0; i < kNumConnections; ++i) { + workers.emplace_back([connect_address_resolved]() { + int client_fd; + int one = 1; + char buf[1024]; + client_fd = socket(AF_INET6, SOCK_STREAM, 0); + setsockopt(client_fd, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one)); + // Connection should succeed. + EXPECT_EQ(connect(client_fd, + const_cast( + connect_address_resolved->address()), + connect_address_resolved->size()), + 0); + // recv should not block forever and it should return because + // GRPC_ARG_SERVER_HANDSHAKE_TIMEOUT_MS is set and the server should + // close this connections after that timeout expires. + while (recv(client_fd, buf, 1024, 0) > 0) { + } + close(client_fd); + }); + } + for (int i = 0; i < kNumConnections; ++i) { + workers[i].join(); + } +#endif + // Subsequent kNumConnections / 2 RPCs should succeed because the previously + // spawned client connections have been closed. std::vector> stubs; stubs.reserve(kNumConnections); for (int i = 0; i < kNumConnections; i++) { @@ -206,6 +255,7 @@ class End2EndConnectionQuotaTest : public ::testing::TestWithParam { int port_; std::unique_ptr server_; string server_address_; + string connect_address_; GrpcCallbackServiceImpl grpc_service_; std::string payload_; }; From 1ce894c977e368ec6eb67f5caf92d4aaaa33144a Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Wed, 13 Mar 2024 14:25:37 -0700 Subject: [PATCH 39/52] [OTel] Add CI support for tests via CMake (#36087) Earlier, the tests just had bazel support. With CMake support added in #36063, we can also add CI CMake support for the tests. A major benefit of this is that we also get coverage for the various platforms that we test from our portability test suite. Closes #36087 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/36087 from yashykt:OTelCISupport b28fbe02e5592b93f9286eb641bd2db25cbe4d9c PiperOrigin-RevId: 615543685 --- CMakeLists.txt | 69 ++++++++++++++++++- build_autogenerated.yaml | 28 ++++++++ templates/CMakeLists.txt.template | 2 +- test/cpp/ext/otel/otel_test_library.h | 2 + .../extract_metadata_from_bazel_xml.py | 3 +- tools/run_tests/generated/tests.json | 24 +++++++ tools/run_tests/helper_scripts/build_cxx.bat | 56 ++++++++++++--- tools/run_tests/helper_scripts/build_cxx.sh | 21 +++++- 8 files changed, 191 insertions(+), 14 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bb8cc07afae..10a84fbc767 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -224,7 +224,7 @@ set(gRPC_ABSL_USED_TARGETS absl_meta ) -# The OpenTelemetry plugin support "package" build only at present. +# The OpenTelemetry plugin supports "package" build only at present. set(gRPC_OPENTELEMETRY_PROVIDER "package") # set(gRPC_OPENTELEMETRY_PROVIDER "module" CACHE STRING "Provider of opentelemetry library") # set_property(CACHE gRPC_OPENTELEMETRY_PROVIDER PROPERTY STRINGS "module" "package") @@ -1215,6 +1215,7 @@ if(gRPC_BUILD_TESTS) add_dependencies(buildtests_cxx orca_service_end2end_test) add_dependencies(buildtests_cxx orphanable_test) add_dependencies(buildtests_cxx osa_distance_test) + add_dependencies(buildtests_cxx otel_plugin_test) add_dependencies(buildtests_cxx out_of_bounds_bad_client_test) add_dependencies(buildtests_cxx outlier_detection_lb_config_parser_test) add_dependencies(buildtests_cxx outlier_detection_test) @@ -20593,6 +20594,72 @@ target_link_libraries(osa_distance_test ) +endif() +if(gRPC_BUILD_TESTS) + +add_executable(otel_plugin_test + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/echo.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/echo.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/echo.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/echo.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/echo_messages.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/echo_messages.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/echo_messages.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/echo_messages.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/simple_messages.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/simple_messages.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/simple_messages.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/simple_messages.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/orca_load_report.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/orca_load_report.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/orca_load_report.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/orca_load_report.grpc.pb.h + src/cpp/ext/otel/otel_client_filter.cc + src/cpp/ext/otel/otel_plugin.cc + src/cpp/ext/otel/otel_server_call_tracer.cc + test/cpp/end2end/test_service_impl.cc + test/cpp/ext/otel/otel_plugin_test.cc + test/cpp/ext/otel/otel_test_library.cc +) +if(WIN32 AND MSVC) + if(BUILD_SHARED_LIBS) + target_compile_definitions(otel_plugin_test + PRIVATE + "GPR_DLL_IMPORTS" + "GRPC_DLL_IMPORTS" + "GRPCXX_DLL_IMPORTS" + ) + endif() +endif() +target_compile_features(otel_plugin_test PUBLIC cxx_std_14) +target_include_directories(otel_plugin_test + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} + third_party/googletest/googletest/include + third_party/googletest/googletest + third_party/googletest/googlemock/include + third_party/googletest/googlemock + ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(otel_plugin_test + ${_gRPC_ALLTARGETS_LIBRARIES} + gtest + opentelemetry-cpp::api + opentelemetry-cpp::metrics + grpc++_test_util +) + + endif() if(gRPC_BUILD_TESTS) diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index 91e82652419..0ef8f449446 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -13206,6 +13206,34 @@ targets: - test/core/util/osa_distance_test.cc deps: - gtest +- name: otel_plugin_test + gtest: true + build: test + language: c++ + headers: + - src/cpp/ext/otel/key_value_iterable.h + - src/cpp/ext/otel/otel_call_tracer.h + - src/cpp/ext/otel/otel_client_filter.h + - src/cpp/ext/otel/otel_plugin.h + - src/cpp/ext/otel/otel_server_call_tracer.h + - test/cpp/end2end/test_service_impl.h + - test/cpp/ext/otel/otel_test_library.h + src: + - src/proto/grpc/testing/echo.proto + - src/proto/grpc/testing/echo_messages.proto + - src/proto/grpc/testing/simple_messages.proto + - src/proto/grpc/testing/xds/v3/orca_load_report.proto + - src/cpp/ext/otel/otel_client_filter.cc + - src/cpp/ext/otel/otel_plugin.cc + - src/cpp/ext/otel/otel_server_call_tracer.cc + - test/cpp/end2end/test_service_impl.cc + - test/cpp/ext/otel/otel_plugin_test.cc + - test/cpp/ext/otel/otel_test_library.cc + deps: + - gtest + - opentelemetry-cpp::api + - opentelemetry-cpp::metrics + - grpc++_test_util - name: out_of_bounds_bad_client_test gtest: true build: test diff --git a/templates/CMakeLists.txt.template b/templates/CMakeLists.txt.template index a5ea02e73ef..3e69ad92258 100644 --- a/templates/CMakeLists.txt.template +++ b/templates/CMakeLists.txt.template @@ -361,7 +361,7 @@ absl_meta ) - # The OpenTelemetry plugin support "package" build only at present. + # The OpenTelemetry plugin supports "package" build only at present. set(gRPC_OPENTELEMETRY_PROVIDER "package") # set(gRPC_OPENTELEMETRY_PROVIDER "module" CACHE STRING "Provider of opentelemetry library") # set_property(CACHE gRPC_OPENTELEMETRY_PROVIDER PROPERTY STRINGS "module" "package") diff --git a/test/cpp/ext/otel/otel_test_library.h b/test/cpp/ext/otel/otel_test_library.h index 8e800d1e30f..11d6d3117a2 100644 --- a/test/cpp/ext/otel/otel_test_library.h +++ b/test/cpp/ext/otel/otel_test_library.h @@ -19,6 +19,8 @@ #ifndef GRPC_TEST_CPP_EXT_OTEL_OTEL_TEST_LIBRARY_H #define GRPC_TEST_CPP_EXT_OTEL_OTEL_TEST_LIBRARY_H +#include + #include "absl/functional/any_invocable.h" #include "gmock/gmock.h" #include "gtest/gtest.h" diff --git a/tools/buildgen/extract_metadata_from_bazel_xml.py b/tools/buildgen/extract_metadata_from_bazel_xml.py index 85b168a9822..bcf57fab2c4 100755 --- a/tools/buildgen/extract_metadata_from_bazel_xml.py +++ b/tools/buildgen/extract_metadata_from_bazel_xml.py @@ -835,8 +835,7 @@ def _exclude_unwanted_cc_tests(tests: List[str]) -> List[str]: tests = [ test for test in tests - if not test.startswith("test/cpp/ext/otel:") - and not test.startswith("test/cpp/ext/csm:") + if not test.startswith("test/cpp/ext/csm:") and not test.startswith("test/cpp/interop:xds_interop") ] diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 49f6bb44395..8965c31bb25 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -6455,6 +6455,30 @@ ], "uses_polling": true }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": true, + "language": "c++", + "name": "otel_plugin_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "uses_polling": true + }, { "args": [], "benchmark": false, diff --git a/tools/run_tests/helper_scripts/build_cxx.bat b/tools/run_tests/helper_scripts/build_cxx.bat index 63c1b641779..2f9d59115f7 100644 --- a/tools/run_tests/helper_scripts/build_cxx.bat +++ b/tools/run_tests/helper_scripts/build_cxx.bat @@ -16,11 +16,6 @@ setlocal cd /d %~dp0\..\..\.. -mkdir cmake -cd cmake -mkdir build -cd build - If "%GRPC_BUILD_ACTIVATE_VS_TOOLS%" == "2019" ( @rem set cl.exe build environment to build with VS2019 tooling @rem this is required for Ninja build to work @@ -49,20 +44,63 @@ If "%GRPC_CMAKE_GENERATOR%" == "Visual Studio 16 2019" ( If "%GRPC_CMAKE_GENERATOR%" == "Ninja" ( @rem Use ninja - @rem Select MSVC compiler (cl.exe) explicitly to make sure we don't end up gcc from mingw or cygwin @rem (both are on path in kokoro win workers) - cmake -G "%GRPC_CMAKE_GENERATOR%" -DCMAKE_C_COMPILER="cl.exe" -DCMAKE_CXX_COMPILER="cl.exe" -DgRPC_BUILD_TESTS=ON -DCMAKE_BUILD_TYPE="%MSBUILD_CONFIG%" %* ../.. || goto :error + + @rem Install abseil-cpp since opentelemetry CMake uses find_package to find it. + cd third_party/abseil-cpp + mkdir build + cd build + cmake -G "%GRPC_CMAKE_GENERATOR%" -DCMAKE_C_COMPILER="cl.exe" -DCMAKE_CXX_COMPILER="cl.exe" -DABSL_BUILD_TESTING=OFF -DCMAKE_BUILD_TYPE="%MSBUILD_CONFIG%" %* .. || goto :error + ninja -j%GRPC_RUN_TESTS_JOBS% install || goto :error + + @rem Install opentelemetry-cpp since we only support "package" mode for opentelemetry at present. + cd ../../.. + cd third_party/opentelemetry-cpp + mkdir build + cd build + cmake -G "%GRPC_CMAKE_GENERATOR%" -DCMAKE_C_COMPILER="cl.exe" -DCMAKE_CXX_COMPILER="cl.exe" -DWITH_ABSEIL=ON -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE="%MSBUILD_CONFIG%" %* .. || goto :error + ninja -j%GRPC_RUN_TESTS_JOBS% install || goto :error + + cd ../../.. + + mkdir cmake + cd cmake + mkdir build + cd build + + cmake -G "%GRPC_CMAKE_GENERATOR%" -DCMAKE_C_COMPILER="cl.exe" -DCMAKE_CXX_COMPILER="cl.exe" -DgRPC_BUILD_GRPCPP_OTEL_PLUGIN=ON -DgRPC_ABSL_PROVIDER=package -DgRPC_BUILD_TESTS=ON -DCMAKE_BUILD_TYPE="%MSBUILD_CONFIG%" %* ../.. || goto :error ninja -j%GRPC_RUN_TESTS_JOBS% buildtests_%GRPC_RUN_TESTS_CXX_LANGUAGE_SUFFIX% || goto :error ) else ( @rem Use one of the Visual Studio generators. - cmake -G "%GRPC_CMAKE_GENERATOR%" -A "%GRPC_CMAKE_ARCHITECTURE%" %CMAKE_SYSTEM_VERSION_ARG% -DCMAKE_VS_PLATFORM_TOOLSET_HOST_ARCHITECTURE=x64 -DgRPC_BUILD_TESTS=ON -DgRPC_BUILD_MSVC_MP_COUNT=%GRPC_RUN_TESTS_JOBS% %* ../.. || goto :error + @rem Install abseil-cpp since opentelemetry CMake uses find_package to find it. + cd third_party/abseil-cpp + mkdir build + cd build + cmake -G "%GRPC_CMAKE_GENERATOR%" -A "%GRPC_CMAKE_ARCHITECTURE%" %CMAKE_SYSTEM_VERSION_ARG% -DCMAKE_VS_PLATFORM_TOOLSET_HOST_ARCHITECTURE=x64 -DABSL_BUILD_TESTING=OFF .. || goto :error + cmake --build . --target install || goto :error + + @rem Install opentelemetry-cpp since we only support "package" mode for opentelemetry at present. + cd ../../.. + cd third_party/opentelemetry-cpp + mkdir build + cd build + cmake -G "%GRPC_CMAKE_GENERATOR%" -A "%GRPC_CMAKE_ARCHITECTURE%" %CMAKE_SYSTEM_VERSION_ARG% -DCMAKE_VS_PLATFORM_TOOLSET_HOST_ARCHITECTURE=x64 -DWITH_ABSEIL=ON -DBUILD_TESTING=OFF .. || goto :error + cmake --build . --target install -j%GRPC_RUN_TESTS_JOBS% || goto :error + + cd ../../.. + mkdir cmake + cd cmake + mkdir build + cd build + + cmake -G "%GRPC_CMAKE_GENERATOR%" -A "%GRPC_CMAKE_ARCHITECTURE%" %CMAKE_SYSTEM_VERSION_ARG% -DCMAKE_VS_PLATFORM_TOOLSET_HOST_ARCHITECTURE=x64 -DgRPC_BUILD_GRPCPP_OTEL_PLUGIN=ON -DgRPC_ABSL_PROVIDER=package -DgRPC_BUILD_TESTS=ON -DgRPC_BUILD_MSVC_MP_COUNT=%GRPC_RUN_TESTS_JOBS% %* ../.. || goto :error @rem GRPC_RUN_TESTS_CXX_LANGUAGE_SUFFIX will be set to either "c" or "cxx" - cmake --build . --target buildtests_%GRPC_RUN_TESTS_CXX_LANGUAGE_SUFFIX% --config %MSBUILD_CONFIG% || goto :error + cmake --build . --target buildtests_%GRPC_RUN_TESTS_CXX_LANGUAGE_SUFFIX% --config %MSBUILD_CONFIG% -j%GRPC_RUN_TESTS_JOBS% || goto :error ) endlocal diff --git a/tools/run_tests/helper_scripts/build_cxx.sh b/tools/run_tests/helper_scripts/build_cxx.sh index 310da2e9ae0..882123e4093 100755 --- a/tools/run_tests/helper_scripts/build_cxx.sh +++ b/tools/run_tests/helper_scripts/build_cxx.sh @@ -15,13 +15,32 @@ set -ex +# Set install path to avoid installing to system paths cd "$(dirname "$0")/../../.." +mkdir -p cmake/install +INSTALL_PATH="$(pwd)/cmake/install" +# Install abseil-cpp since opentelemetry CMake uses find_package to find it. +cd third_party/abseil-cpp +mkdir build +cd build +cmake -DCMAKE_CXX_STANDARD=14 -DABSL_BUILD_TESTING=OFF -DCMAKE_BUILD_TYPE="${MSBUILD_CONFIG}" -DCMAKE_INSTALL_PREFIX="${INSTALL_PATH}" "$@" .. +make -j"${GRPC_RUN_TESTS_JOBS}" install + +# Install opentelemetry-cpp since we only support "package" mode for opentelemetry at present. +cd ../../.. +cd third_party/opentelemetry-cpp +mkdir build +cd build +cmake -DCMAKE_CXX_STANDARD=14 -DWITH_ABSEIL=ON -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE="${MSBUILD_CONFIG}" -DCMAKE_INSTALL_PREFIX="${INSTALL_PATH}" "$@" .. +make -j"${GRPC_RUN_TESTS_JOBS}" install + +cd ../../.. mkdir -p cmake/build cd cmake/build # MSBUILD_CONFIG's values are suitable for cmake as well -cmake -DgRPC_BUILD_TESTS=ON -DCMAKE_BUILD_TYPE="${MSBUILD_CONFIG}" "$@" ../.. +cmake -DCMAKE_CXX_STANDARD=14 -DgRPC_BUILD_GRPCPP_OTEL_PLUGIN=ON -DgRPC_ABSL_PROVIDER=package -DgRPC_BUILD_TESTS=ON -DCMAKE_BUILD_TYPE="${MSBUILD_CONFIG}" -DCMAKE_INSTALL_PREFIX="${INSTALL_PATH}" "$@" ../.. # GRPC_RUN_TESTS_CXX_LANGUAGE_SUFFIX will be set to either "c" or "cxx" make -j"${GRPC_RUN_TESTS_JOBS}" "buildtests_${GRPC_RUN_TESTS_CXX_LANGUAGE_SUFFIX}" "tools_${GRPC_RUN_TESTS_CXX_LANGUAGE_SUFFIX}" From 424e95ccbb2099242a2c71b2d0154eb537b3cb41 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Wed, 13 Mar 2024 15:33:16 -0700 Subject: [PATCH 40/52] [Max Message Limits] Improve logging to determine client/server (#36103) Will help debugging errors like #35805 Closes #36103 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/36103 from yashykt:HelpDebugMessageSizeLimits 5507a257f39d7264e8ccc838437f97a2602c6d19 PiperOrigin-RevId: 615564790 --- .../message_compress/compression_filter.cc | 15 ++++++------ .../message_compress/compression_filter.h | 3 ++- .../legacy_compression_filter.cc | 13 +++++++---- .../legacy_compression_filter.h | 3 ++- .../message_size/message_size_filter.cc | 23 +++++++++++-------- .../tests/InteropTests/InteropTests.m | 2 +- test/core/end2end/tests/max_message_length.cc | 14 ++++++----- 7 files changed, 43 insertions(+), 30 deletions(-) diff --git a/src/core/ext/filters/http/message_compress/compression_filter.cc b/src/core/ext/filters/http/message_compress/compression_filter.cc index 1ffee085230..c9b3acaf1f7 100644 --- a/src/core/ext/filters/http/message_compress/compression_filter.cc +++ b/src/core/ext/filters/http/message_compress/compression_filter.cc @@ -169,7 +169,7 @@ MessageHandle ChannelCompression::CompressMessage( } absl::StatusOr ChannelCompression::DecompressMessage( - MessageHandle message, DecompressArgs args) const { + bool is_client, MessageHandle message, DecompressArgs args) const { if (GRPC_TRACE_FLAG_ENABLED(grpc_compression_trace)) { gpr_log(GPR_INFO, "DecompressMessage: len=%" PRIdPTR " max=%d alg=%d", message->payload()->Length(), @@ -186,8 +186,9 @@ absl::StatusOr ChannelCompression::DecompressMessage( message->payload()->Length() > static_cast(*args.max_recv_message_length)) { return absl::ResourceExhaustedError(absl::StrFormat( - "Received message larger than max (%u vs. %d)", - message->payload()->Length(), *args.max_recv_message_length)); + "%s: Received message larger than max (%u vs. %d)", + is_client ? "CLIENT" : "SERVER", message->payload()->Length(), + *args.max_recv_message_length)); } // Check if decompression is enabled (if not, we can just pass the message // up). @@ -264,8 +265,8 @@ void ClientCompressionFilter::Call::OnServerInitialMetadata( absl::StatusOr ClientCompressionFilter::Call::OnServerToClientMessage( MessageHandle message, ClientCompressionFilter* filter) { - return filter->compression_engine_.DecompressMessage(std::move(message), - decompress_args_); + return filter->compression_engine_.DecompressMessage( + /*is_client=*/true, std::move(message), decompress_args_); } void ServerCompressionFilter::Call::OnClientInitialMetadata( @@ -276,8 +277,8 @@ void ServerCompressionFilter::Call::OnClientInitialMetadata( absl::StatusOr ServerCompressionFilter::Call::OnClientToServerMessage( MessageHandle message, ServerCompressionFilter* filter) { - return filter->compression_engine_.DecompressMessage(std::move(message), - decompress_args_); + return filter->compression_engine_.DecompressMessage( + /*is_client=*/false, std::move(message), decompress_args_); } void ServerCompressionFilter::Call::OnServerInitialMetadata( diff --git a/src/core/ext/filters/http/message_compress/compression_filter.h b/src/core/ext/filters/http/message_compress/compression_filter.h index 40e88d9b5a8..adf9fb49355 100644 --- a/src/core/ext/filters/http/message_compress/compression_filter.h +++ b/src/core/ext/filters/http/message_compress/compression_filter.h @@ -87,7 +87,8 @@ class ChannelCompression { MessageHandle CompressMessage(MessageHandle message, grpc_compression_algorithm algorithm) const; // Decompress one message synchronously. - absl::StatusOr DecompressMessage(MessageHandle message, + absl::StatusOr DecompressMessage(bool is_client, + MessageHandle message, DecompressArgs args) const; private: diff --git a/src/core/ext/filters/http/message_compress/legacy_compression_filter.cc b/src/core/ext/filters/http/message_compress/legacy_compression_filter.cc index 8f62d610b33..9fe6746a48d 100644 --- a/src/core/ext/filters/http/message_compress/legacy_compression_filter.cc +++ b/src/core/ext/filters/http/message_compress/legacy_compression_filter.cc @@ -166,7 +166,7 @@ MessageHandle LegacyCompressionFilter::CompressMessage( } absl::StatusOr LegacyCompressionFilter::DecompressMessage( - MessageHandle message, DecompressArgs args) const { + bool is_client, MessageHandle message, DecompressArgs args) const { if (GRPC_TRACE_FLAG_ENABLED(grpc_compression_trace)) { gpr_log(GPR_INFO, "DecompressMessage: len=%" PRIdPTR " max=%d alg=%d", message->payload()->Length(), @@ -183,8 +183,9 @@ absl::StatusOr LegacyCompressionFilter::DecompressMessage( message->payload()->Length() > static_cast(*args.max_recv_message_length)) { return absl::ResourceExhaustedError(absl::StrFormat( - "Received message larger than max (%u vs. %d)", - message->payload()->Length(), *args.max_recv_message_length)); + "%s: Received message larger than max (%u vs. %d)", + is_client ? "CLIENT" : "SERVER", message->payload()->Length(), + *args.max_recv_message_length)); } // Check if decompression is enabled (if not, we can just pass the message // up). @@ -266,7 +267,8 @@ LegacyClientCompressionFilter::MakeCallPromise( call_args.server_to_client_messages->InterceptAndMap( [decompress_err, decompress_args, this](MessageHandle message) -> absl::optional { - auto r = DecompressMessage(std::move(message), *decompress_args); + auto r = DecompressMessage(/*is_client=*/true, std::move(message), + *decompress_args); if (!r.ok()) { decompress_err->Set(ServerMetadataFromStatus(r.status())); return absl::nullopt; @@ -288,7 +290,8 @@ LegacyServerCompressionFilter::MakeCallPromise( call_args.client_to_server_messages->InterceptAndMap( [decompress_err, decompress_args, this](MessageHandle message) -> absl::optional { - auto r = DecompressMessage(std::move(message), decompress_args); + auto r = DecompressMessage(/*is_client=*/false, std::move(message), + decompress_args); if (grpc_call_trace.enabled()) { gpr_log(GPR_DEBUG, "%s[compression] DecompressMessage returned %s", GetContext()->DebugTag().c_str(), diff --git a/src/core/ext/filters/http/message_compress/legacy_compression_filter.h b/src/core/ext/filters/http/message_compress/legacy_compression_filter.h index 0926bc09ed9..f2212e2d61c 100644 --- a/src/core/ext/filters/http/message_compress/legacy_compression_filter.h +++ b/src/core/ext/filters/http/message_compress/legacy_compression_filter.h @@ -87,7 +87,8 @@ class LegacyCompressionFilter : public ChannelFilter { MessageHandle CompressMessage(MessageHandle message, grpc_compression_algorithm algorithm) const; // Decompress one message synchronously. - absl::StatusOr DecompressMessage(MessageHandle message, + absl::StatusOr DecompressMessage(bool is_client, + MessageHandle message, DecompressArgs args) const; private: diff --git a/src/core/ext/filters/message_size/message_size_filter.cc b/src/core/ext/filters/message_size/message_size_filter.cc index c9fe96cc41c..df1b4b53306 100644 --- a/src/core/ext/filters/message_size/message_size_filter.cc +++ b/src/core/ext/filters/message_size/message_size_filter.cc @@ -160,7 +160,7 @@ absl::StatusOr ServerMessageSizeFilter::Create( namespace { ServerMetadataHandle CheckPayload(const Message& msg, absl::optional max_length, - bool is_send) { + bool is_client, bool is_send) { if (!max_length.has_value()) return nullptr; if (GRPC_TRACE_FLAG_ENABLED(grpc_call_trace)) { gpr_log(GPR_INFO, "%s[message_size] %s len:%" PRIdPTR " max:%d", @@ -170,10 +170,11 @@ ServerMetadataHandle CheckPayload(const Message& msg, if (msg.payload()->Length() <= *max_length) return nullptr; auto r = GetContext()->MakePooled(GetContext()); r->Set(GrpcStatusMetadata(), GRPC_STATUS_RESOURCE_EXHAUSTED); - r->Set(GrpcMessageMetadata(), Slice::FromCopiedString(absl::StrFormat( - "%s message larger than max (%u vs. %d)", - is_send ? "Sent" : "Received", - msg.payload()->Length(), *max_length))); + r->Set(GrpcMessageMetadata(), + Slice::FromCopiedString(absl::StrFormat( + "%s: %s message larger than max (%u vs. %d)", + is_client ? "CLIENT" : "SERVER", is_send ? "Sent" : "Received", + msg.payload()->Length(), *max_length))); return r; } } // namespace @@ -207,22 +208,26 @@ ClientMessageSizeFilter::Call::Call(ClientMessageSizeFilter* filter) ServerMetadataHandle ServerMessageSizeFilter::Call::OnClientToServerMessage( const Message& message, ServerMessageSizeFilter* filter) { - return CheckPayload(message, filter->parsed_config_.max_recv_size(), false); + return CheckPayload(message, filter->parsed_config_.max_recv_size(), + /*is_client=*/false, false); } ServerMetadataHandle ServerMessageSizeFilter::Call::OnServerToClientMessage( const Message& message, ServerMessageSizeFilter* filter) { - return CheckPayload(message, filter->parsed_config_.max_send_size(), true); + return CheckPayload(message, filter->parsed_config_.max_send_size(), + /*is_client=*/false, true); } ServerMetadataHandle ClientMessageSizeFilter::Call::OnClientToServerMessage( const Message& message) { - return CheckPayload(message, limits_.max_send_size(), true); + return CheckPayload(message, limits_.max_send_size(), /*is_client=*/true, + true); } ServerMetadataHandle ClientMessageSizeFilter::Call::OnServerToClientMessage( const Message& message) { - return CheckPayload(message, limits_.max_recv_size(), false); + return CheckPayload(message, limits_.max_recv_size(), /*is_client=*/true, + false); } namespace { diff --git a/src/objective-c/tests/InteropTests/InteropTests.m b/src/objective-c/tests/InteropTests/InteropTests.m index 24122b9a30f..2e6ddf69dd8 100644 --- a/src/objective-c/tests/InteropTests/InteropTests.m +++ b/src/objective-c/tests/InteropTests/InteropTests.m @@ -958,7 +958,7 @@ static dispatch_once_t initGlobalInterceptorFactory; // https://github.com/protocolbuffers/protobuf/blob/master/src/google/protobuf/field_mask.proto XCTAssertEqualObjects( error.localizedDescription, - @"Received message larger than max (4194305 vs. 4194304)"); + @"CLIENT: Received message larger than max (4194305 vs. 4194304)"); [expectation fulfill]; }]; waiterBlock(@[ expectation ], GRPCInteropTestTimeoutDefault); diff --git a/test/core/end2end/tests/max_message_length.cc b/test/core/end2end/tests/max_message_length.cc index 9cabffd6d68..f57963dc725 100644 --- a/test/core/end2end/tests/max_message_length.cc +++ b/test/core/end2end/tests/max_message_length.cc @@ -45,7 +45,8 @@ void TestMaxMessageLengthOnClientOnRequest(CoreEnd2endTest& test) { test.Expect(1, true); test.Step(); EXPECT_EQ(server_status.status(), GRPC_STATUS_RESOURCE_EXHAUSTED); - EXPECT_EQ(server_status.message(), "Sent message larger than max (11 vs. 5)"); + EXPECT_EQ(server_status.message(), + "CLIENT: Sent message larger than max (11 vs. 5)"); } void TestMaxMessageLengthOnServerOnRequest(CoreEnd2endTest& test) { @@ -71,7 +72,7 @@ void TestMaxMessageLengthOnServerOnRequest(CoreEnd2endTest& test) { EXPECT_TRUE(client_close.was_cancelled()); EXPECT_EQ(server_status.status(), GRPC_STATUS_RESOURCE_EXHAUSTED); EXPECT_EQ(server_status.message(), - "Received message larger than max (11 vs. 5)"); + "SERVER: Received message larger than max (11 vs. 5)"); } void TestMaxMessageLengthOnClientOnResponse(CoreEnd2endTest& test) { @@ -100,7 +101,7 @@ void TestMaxMessageLengthOnClientOnResponse(CoreEnd2endTest& test) { EXPECT_EQ(s.method(), "/service/method"); EXPECT_EQ(server_status.status(), GRPC_STATUS_RESOURCE_EXHAUSTED); EXPECT_EQ(server_status.message(), - "Received message larger than max (11 vs. 5)"); + "CLIENT: Received message larger than max (11 vs. 5)"); } void TestMaxMessageLengthOnServerOnResponse(CoreEnd2endTest& test) { @@ -128,7 +129,8 @@ void TestMaxMessageLengthOnServerOnResponse(CoreEnd2endTest& test) { test.Step(); EXPECT_EQ(s.method(), "/service/method"); EXPECT_EQ(server_status.status(), GRPC_STATUS_RESOURCE_EXHAUSTED); - EXPECT_EQ(server_status.message(), "Sent message larger than max (11 vs. 5)"); + EXPECT_EQ(server_status.message(), + "SERVER: Sent message larger than max (11 vs. 5)"); } CORE_END2END_TEST(CoreEnd2endTest, @@ -276,7 +278,7 @@ CORE_END2END_TEST(Http2Test, MaxMessageLengthOnServerOnRequestWithCompression) { EXPECT_TRUE(client_close.was_cancelled()); EXPECT_EQ(server_status.status(), GRPC_STATUS_RESOURCE_EXHAUSTED); EXPECT_THAT(server_status.message(), - StartsWith("Received message larger than max")); + StartsWith("SERVER: Received message larger than max")); } CORE_END2END_TEST(Http2Test, @@ -311,7 +313,7 @@ CORE_END2END_TEST(Http2Test, EXPECT_EQ(s.method(), "/service/method"); EXPECT_EQ(server_status.status(), GRPC_STATUS_RESOURCE_EXHAUSTED); EXPECT_THAT(server_status.message(), - StartsWith("Received message larger than max")); + StartsWith("CLIENT: Received message larger than max")); } } // namespace From 2c49416713b06639dfc885265750cd52f037aae2 Mon Sep 17 00:00:00 2001 From: Tanvi Jagtap <139093547+tanvi-jagtap@users.noreply.github.com> Date: Wed, 13 Mar 2024 22:02:00 -0700 Subject: [PATCH 41/52] [grpc][Gpr_To_Absl_Logging] Using absl from within gpr logging (#36108) ![Config_Vars_Absl (1)](https://github.com/grpc/grpc/assets/139093547/04d9b75c-dd1c-42ee-81a3-d20e2bbd5672) Closes #36108 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/36108 from tanvi-jagtap:tjagtap_config_vars c48a3524f092c88b90ca06e457926127e7bbb21f PiperOrigin-RevId: 615652000 --- BUILD | 2 ++ CMakeLists.txt | 35 +++++++++++++++--- Makefile | 13 +++++++ build_autogenerated.yaml | 2 ++ config.m4 | 15 ++++++++ config.w32 | 15 ++++++++ gRPC-C++.podspec | 2 ++ gRPC-Core.podspec | 2 ++ grpc.gemspec | 36 +++++++++++++++++++ grpc.gyp | 2 ++ package.xml | 36 +++++++++++++++++++ src/core/lib/config/config_vars.cc | 10 +++--- src/core/lib/config/config_vars.h | 4 +++ src/core/lib/config/config_vars.yaml | 5 +++ src/core/lib/gpr/android/log.cc | 2 +- src/core/lib/gpr/linux/log.cc | 2 +- src/core/lib/gpr/log.cc | 26 ++++++++++++++ src/core/lib/gpr/posix/log.cc | 2 +- src/core/lib/gpr/windows/log.cc | 2 +- src/python/grpcio/grpc_core_dependencies.py | 13 +++++++ .../observability_lib_deps.py | 13 +++++++ test/core/util/fuzz_config_vars.cc | 4 --- 22 files changed, 226 insertions(+), 17 deletions(-) diff --git a/BUILD b/BUILD index 617bc1cf533..747f134371c 100644 --- a/BUILD +++ b/BUILD @@ -746,7 +746,9 @@ grpc_cc_library( external_deps = [ "absl/base", "absl/base:core_headers", + "absl/base:log_severity", "absl/functional:any_invocable", + "absl/log", "absl/memory", "absl/random", "absl/status", diff --git a/CMakeLists.txt b/CMakeLists.txt index 10a84fbc767..afaffdda098 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -97,6 +97,7 @@ set(gRPC_ABSL_PROVIDER "module" CACHE STRING "Provider of absl library") set_property(CACHE gRPC_ABSL_PROVIDER PROPERTY STRINGS "module" "package") set(gRPC_ABSL_USED_TARGETS + absl_absl_vlog_is_on absl_algorithm absl_algorithm_container absl_any_invocable @@ -135,6 +136,7 @@ set(gRPC_ABSL_USED_TARGETS absl_dynamic_annotations absl_endian absl_errno_saver + absl_examine_stack absl_exponential_biased absl_fast_type_id absl_fixed_array @@ -163,7 +165,26 @@ set(gRPC_ABSL_USED_TARGETS absl_int128 absl_kernel_timeout_internal absl_layout + absl_log + absl_log_entry + absl_log_globals + absl_log_internal_append_truncated + absl_log_internal_conditions + absl_log_internal_config + absl_log_internal_fnmatch + absl_log_internal_format + absl_log_internal_globals + absl_log_internal_log_impl + absl_log_internal_log_sink_set + absl_log_internal_message + absl_log_internal_nullguard + absl_log_internal_nullstream + absl_log_internal_proto + absl_log_internal_strip + absl_log_internal_voidify absl_log_severity + absl_log_sink + absl_log_sink_registry absl_low_level_hash absl_malloc_internal absl_memory @@ -221,6 +242,8 @@ set(gRPC_ABSL_USED_TARGETS absl_type_traits absl_utility absl_variant + absl_vlog_config_internal + absl_vlog_is_on absl_meta ) @@ -1714,9 +1737,11 @@ target_link_libraries(gpr ${_gRPC_ALLTARGETS_LIBRARIES} absl::base absl::core_headers + absl::log_severity absl::flags absl::flags_marshalling absl::any_invocable + absl::log absl::memory absl::random_random absl::status @@ -36369,7 +36394,7 @@ generate_pkgconfig( "gpr" "gRPC platform support library" "${gRPC_CORE_VERSION}" - "absl_any_invocable absl_base absl_cord absl_core_headers absl_flags absl_flags_marshalling absl_memory absl_optional absl_random_random absl_status absl_str_format absl_strings absl_synchronization absl_time absl_variant" + "absl_any_invocable absl_base absl_cord absl_core_headers absl_flags absl_flags_marshalling absl_log absl_log_severity absl_memory absl_optional absl_random_random absl_status absl_str_format absl_strings absl_synchronization absl_time absl_variant" "" "-lgpr" "" @@ -36380,7 +36405,7 @@ generate_pkgconfig( "gRPC" "high performance general RPC framework" "${gRPC_CORE_VERSION}" - "absl_algorithm_container absl_any_invocable absl_base absl_bind_front absl_cleanup absl_config absl_cord absl_core_headers absl_flags absl_flags_marshalling absl_flat_hash_map absl_flat_hash_set absl_function_ref absl_hash absl_inlined_vector absl_memory absl_no_destructor absl_optional absl_random_bit_gen_ref absl_random_distributions absl_random_random absl_span absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time absl_type_traits absl_utility absl_variant gpr" + "absl_algorithm_container absl_any_invocable absl_base absl_bind_front absl_cleanup absl_config absl_cord absl_core_headers absl_flags absl_flags_marshalling absl_flat_hash_map absl_flat_hash_set absl_function_ref absl_hash absl_inlined_vector absl_log absl_log_severity absl_memory absl_no_destructor absl_optional absl_random_bit_gen_ref absl_random_distributions absl_random_random absl_span absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time absl_type_traits absl_utility absl_variant gpr" "libcares openssl re2 zlib" "-lgrpc" "-laddress_sorting -lupb_textformat_lib -lupb_json_lib -lutf8_range_lib -lupb_message_lib -lupb_mem_lib -lupb_base_lib" @@ -36391,7 +36416,7 @@ generate_pkgconfig( "gRPC unsecure" "high performance general RPC framework without SSL" "${gRPC_CORE_VERSION}" - "absl_algorithm_container absl_any_invocable absl_base absl_bind_front absl_cleanup absl_config absl_cord absl_core_headers absl_flags absl_flags_marshalling absl_flat_hash_map absl_flat_hash_set absl_function_ref absl_hash absl_inlined_vector absl_memory absl_no_destructor absl_optional absl_random_bit_gen_ref absl_random_distributions absl_random_random absl_span absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time absl_type_traits absl_utility absl_variant gpr" + "absl_algorithm_container absl_any_invocable absl_base absl_bind_front absl_cleanup absl_config absl_cord absl_core_headers absl_flags absl_flags_marshalling absl_flat_hash_map absl_flat_hash_set absl_function_ref absl_hash absl_inlined_vector absl_log absl_log_severity absl_memory absl_no_destructor absl_optional absl_random_bit_gen_ref absl_random_distributions absl_random_random absl_span absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time absl_type_traits absl_utility absl_variant gpr" "libcares zlib" "-lgrpc_unsecure" "-laddress_sorting -lutf8_range_lib -lupb_message_lib -lupb_mem_lib -lupb_base_lib" @@ -36402,7 +36427,7 @@ generate_pkgconfig( "gRPC++" "C++ wrapper for gRPC" "${gRPC_CPP_VERSION}" - "absl_algorithm_container absl_any_invocable absl_base absl_bind_front absl_cleanup absl_config absl_cord absl_core_headers absl_flags absl_flags_marshalling absl_flat_hash_map absl_flat_hash_set absl_function_ref absl_hash absl_inlined_vector absl_memory absl_no_destructor absl_optional absl_random_bit_gen_ref absl_random_distributions absl_random_random absl_span absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time absl_type_traits absl_utility absl_variant gpr grpc" + "absl_algorithm_container absl_any_invocable absl_base absl_bind_front absl_cleanup absl_config absl_cord absl_core_headers absl_flags absl_flags_marshalling absl_flat_hash_map absl_flat_hash_set absl_function_ref absl_hash absl_inlined_vector absl_log absl_log_severity absl_memory absl_no_destructor absl_optional absl_random_bit_gen_ref absl_random_distributions absl_random_random absl_span absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time absl_type_traits absl_utility absl_variant gpr grpc" "libcares openssl re2 zlib" "-lgrpc++" "-laddress_sorting -lupb_textformat_lib -lupb_json_lib -lutf8_range_lib -lupb_message_lib -lupb_mem_lib -lupb_base_lib" @@ -36413,7 +36438,7 @@ generate_pkgconfig( "gRPC++ unsecure" "C++ wrapper for gRPC without SSL" "${gRPC_CPP_VERSION}" - "absl_algorithm_container absl_any_invocable absl_base absl_bind_front absl_cleanup absl_config absl_cord absl_core_headers absl_flags absl_flags_marshalling absl_flat_hash_map absl_flat_hash_set absl_function_ref absl_hash absl_inlined_vector absl_memory absl_no_destructor absl_optional absl_random_bit_gen_ref absl_random_distributions absl_random_random absl_span absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time absl_type_traits absl_utility absl_variant gpr grpc_unsecure" + "absl_algorithm_container absl_any_invocable absl_base absl_bind_front absl_cleanup absl_config absl_cord absl_core_headers absl_flags absl_flags_marshalling absl_flat_hash_map absl_flat_hash_set absl_function_ref absl_hash absl_inlined_vector absl_log absl_log_severity absl_memory absl_no_destructor absl_optional absl_random_bit_gen_ref absl_random_distributions absl_random_random absl_span absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time absl_type_traits absl_utility absl_variant gpr grpc_unsecure" "libcares zlib" "-lgrpc++_unsecure" "-laddress_sorting -lutf8_range_lib -lupb_message_lib -lupb_mem_lib -lupb_base_lib" diff --git a/Makefile b/Makefile index 23f195b095a..a6fb5f7ddc3 100644 --- a/Makefile +++ b/Makefile @@ -1539,6 +1539,7 @@ LIBGRPC_SRC = \ third_party/abseil-cpp/absl/debugging/internal/address_is_readable.cc \ third_party/abseil-cpp/absl/debugging/internal/demangle.cc \ third_party/abseil-cpp/absl/debugging/internal/elf_mem_image.cc \ + third_party/abseil-cpp/absl/debugging/internal/examine_stack.cc \ third_party/abseil-cpp/absl/debugging/internal/vdso_support.cc \ third_party/abseil-cpp/absl/debugging/stacktrace.cc \ third_party/abseil-cpp/absl/debugging/symbolize.cc \ @@ -1553,6 +1554,18 @@ LIBGRPC_SRC = \ third_party/abseil-cpp/absl/hash/internal/city.cc \ third_party/abseil-cpp/absl/hash/internal/hash.cc \ third_party/abseil-cpp/absl/hash/internal/low_level_hash.cc \ + third_party/abseil-cpp/absl/log/globals.cc \ + third_party/abseil-cpp/absl/log/internal/conditions.cc \ + third_party/abseil-cpp/absl/log/internal/fnmatch.cc \ + third_party/abseil-cpp/absl/log/internal/globals.cc \ + third_party/abseil-cpp/absl/log/internal/log_format.cc \ + third_party/abseil-cpp/absl/log/internal/log_message.cc \ + third_party/abseil-cpp/absl/log/internal/log_sink_set.cc \ + third_party/abseil-cpp/absl/log/internal/nullguard.cc \ + third_party/abseil-cpp/absl/log/internal/proto.cc \ + third_party/abseil-cpp/absl/log/internal/vlog_config.cc \ + third_party/abseil-cpp/absl/log/log_entry.cc \ + third_party/abseil-cpp/absl/log/log_sink.cc \ third_party/abseil-cpp/absl/numeric/int128.cc \ third_party/abseil-cpp/absl/profiling/internal/exponential_biased.cc \ third_party/abseil-cpp/absl/random/discrete_distribution.cc \ diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index 0ef8f449446..8f3765d9398 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -125,9 +125,11 @@ libs: deps: - absl/base:base - absl/base:core_headers + - absl/base:log_severity - absl/flags:flag - absl/flags:marshalling - absl/functional:any_invocable + - absl/log:log - absl/memory:memory - absl/random:random - absl/status:status diff --git a/config.m4 b/config.m4 index 71180188c82..0e5da21558c 100644 --- a/config.m4 +++ b/config.m4 @@ -924,6 +924,7 @@ if test "$PHP_GRPC" != "no"; then third_party/abseil-cpp/absl/debugging/internal/address_is_readable.cc \ third_party/abseil-cpp/absl/debugging/internal/demangle.cc \ third_party/abseil-cpp/absl/debugging/internal/elf_mem_image.cc \ + third_party/abseil-cpp/absl/debugging/internal/examine_stack.cc \ third_party/abseil-cpp/absl/debugging/internal/vdso_support.cc \ third_party/abseil-cpp/absl/debugging/stacktrace.cc \ third_party/abseil-cpp/absl/debugging/symbolize.cc \ @@ -938,6 +939,18 @@ if test "$PHP_GRPC" != "no"; then third_party/abseil-cpp/absl/hash/internal/city.cc \ third_party/abseil-cpp/absl/hash/internal/hash.cc \ third_party/abseil-cpp/absl/hash/internal/low_level_hash.cc \ + third_party/abseil-cpp/absl/log/globals.cc \ + third_party/abseil-cpp/absl/log/internal/conditions.cc \ + third_party/abseil-cpp/absl/log/internal/fnmatch.cc \ + third_party/abseil-cpp/absl/log/internal/globals.cc \ + third_party/abseil-cpp/absl/log/internal/log_format.cc \ + third_party/abseil-cpp/absl/log/internal/log_message.cc \ + third_party/abseil-cpp/absl/log/internal/log_sink_set.cc \ + third_party/abseil-cpp/absl/log/internal/nullguard.cc \ + third_party/abseil-cpp/absl/log/internal/proto.cc \ + third_party/abseil-cpp/absl/log/internal/vlog_config.cc \ + third_party/abseil-cpp/absl/log/log_entry.cc \ + third_party/abseil-cpp/absl/log/log_sink.cc \ third_party/abseil-cpp/absl/numeric/int128.cc \ third_party/abseil-cpp/absl/profiling/internal/exponential_biased.cc \ third_party/abseil-cpp/absl/random/discrete_distribution.cc \ @@ -1600,6 +1613,8 @@ if test "$PHP_GRPC" != "no"; then PHP_ADD_BUILD_DIR($ext_builddir/third_party/abseil-cpp/absl/flags) PHP_ADD_BUILD_DIR($ext_builddir/third_party/abseil-cpp/absl/flags/internal) PHP_ADD_BUILD_DIR($ext_builddir/third_party/abseil-cpp/absl/hash/internal) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/abseil-cpp/absl/log) + PHP_ADD_BUILD_DIR($ext_builddir/third_party/abseil-cpp/absl/log/internal) PHP_ADD_BUILD_DIR($ext_builddir/third_party/abseil-cpp/absl/numeric) PHP_ADD_BUILD_DIR($ext_builddir/third_party/abseil-cpp/absl/profiling/internal) PHP_ADD_BUILD_DIR($ext_builddir/third_party/abseil-cpp/absl/random) diff --git a/config.w32 b/config.w32 index b9e6c920b2e..0dea3cdcccf 100644 --- a/config.w32 +++ b/config.w32 @@ -889,6 +889,7 @@ if (PHP_GRPC != "no") { "third_party\\abseil-cpp\\absl\\debugging\\internal\\address_is_readable.cc " + "third_party\\abseil-cpp\\absl\\debugging\\internal\\demangle.cc " + "third_party\\abseil-cpp\\absl\\debugging\\internal\\elf_mem_image.cc " + + "third_party\\abseil-cpp\\absl\\debugging\\internal\\examine_stack.cc " + "third_party\\abseil-cpp\\absl\\debugging\\internal\\vdso_support.cc " + "third_party\\abseil-cpp\\absl\\debugging\\stacktrace.cc " + "third_party\\abseil-cpp\\absl\\debugging\\symbolize.cc " + @@ -903,6 +904,18 @@ if (PHP_GRPC != "no") { "third_party\\abseil-cpp\\absl\\hash\\internal\\city.cc " + "third_party\\abseil-cpp\\absl\\hash\\internal\\hash.cc " + "third_party\\abseil-cpp\\absl\\hash\\internal\\low_level_hash.cc " + + "third_party\\abseil-cpp\\absl\\log\\globals.cc " + + "third_party\\abseil-cpp\\absl\\log\\internal\\conditions.cc " + + "third_party\\abseil-cpp\\absl\\log\\internal\\fnmatch.cc " + + "third_party\\abseil-cpp\\absl\\log\\internal\\globals.cc " + + "third_party\\abseil-cpp\\absl\\log\\internal\\log_format.cc " + + "third_party\\abseil-cpp\\absl\\log\\internal\\log_message.cc " + + "third_party\\abseil-cpp\\absl\\log\\internal\\log_sink_set.cc " + + "third_party\\abseil-cpp\\absl\\log\\internal\\nullguard.cc " + + "third_party\\abseil-cpp\\absl\\log\\internal\\proto.cc " + + "third_party\\abseil-cpp\\absl\\log\\internal\\vlog_config.cc " + + "third_party\\abseil-cpp\\absl\\log\\log_entry.cc " + + "third_party\\abseil-cpp\\absl\\log\\log_sink.cc " + "third_party\\abseil-cpp\\absl\\numeric\\int128.cc " + "third_party\\abseil-cpp\\absl\\profiling\\internal\\exponential_biased.cc " + "third_party\\abseil-cpp\\absl\\random\\discrete_distribution.cc " + @@ -1747,6 +1760,8 @@ if (PHP_GRPC != "no") { FSO.CreateFolder(base_dir+"\\ext\\grpc\\third_party\\abseil-cpp\\absl\\flags\\internal"); FSO.CreateFolder(base_dir+"\\ext\\grpc\\third_party\\abseil-cpp\\absl\\hash"); FSO.CreateFolder(base_dir+"\\ext\\grpc\\third_party\\abseil-cpp\\absl\\hash\\internal"); + FSO.CreateFolder(base_dir+"\\ext\\grpc\\third_party\\abseil-cpp\\absl\\log"); + FSO.CreateFolder(base_dir+"\\ext\\grpc\\third_party\\abseil-cpp\\absl\\log\\internal"); FSO.CreateFolder(base_dir+"\\ext\\grpc\\third_party\\abseil-cpp\\absl\\numeric"); FSO.CreateFolder(base_dir+"\\ext\\grpc\\third_party\\abseil-cpp\\absl\\profiling"); FSO.CreateFolder(base_dir+"\\ext\\grpc\\third_party\\abseil-cpp\\absl\\profiling\\internal"); diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index f459d1b7b85..b8e6e3d892b 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -230,6 +230,7 @@ Pod::Spec.new do |s| ss.dependency 'abseil/base/base', abseil_version ss.dependency 'abseil/base/config', abseil_version ss.dependency 'abseil/base/core_headers', abseil_version + ss.dependency 'abseil/base/log_severity', abseil_version ss.dependency 'abseil/base/no_destructor', abseil_version ss.dependency 'abseil/cleanup/cleanup', abseil_version ss.dependency 'abseil/container/flat_hash_map', abseil_version @@ -241,6 +242,7 @@ Pod::Spec.new do |s| ss.dependency 'abseil/functional/bind_front', abseil_version ss.dependency 'abseil/functional/function_ref', abseil_version ss.dependency 'abseil/hash/hash', abseil_version + ss.dependency 'abseil/log/log', abseil_version ss.dependency 'abseil/memory/memory', abseil_version ss.dependency 'abseil/meta/type_traits', abseil_version ss.dependency 'abseil/random/bit_gen_ref', abseil_version diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index dd30ea8829c..67f256a60e3 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -200,6 +200,7 @@ Pod::Spec.new do |s| ss.dependency 'abseil/base/base', abseil_version ss.dependency 'abseil/base/config', abseil_version ss.dependency 'abseil/base/core_headers', abseil_version + ss.dependency 'abseil/base/log_severity', abseil_version ss.dependency 'abseil/base/no_destructor', abseil_version ss.dependency 'abseil/cleanup/cleanup', abseil_version ss.dependency 'abseil/container/flat_hash_map', abseil_version @@ -211,6 +212,7 @@ Pod::Spec.new do |s| ss.dependency 'abseil/functional/bind_front', abseil_version ss.dependency 'abseil/functional/function_ref', abseil_version ss.dependency 'abseil/hash/hash', abseil_version + ss.dependency 'abseil/log/log', abseil_version ss.dependency 'abseil/memory/memory', abseil_version ss.dependency 'abseil/meta/type_traits', abseil_version ss.dependency 'abseil/random/bit_gen_ref', abseil_version diff --git a/grpc.gemspec b/grpc.gemspec index 64b5cf62e4b..1016f7ed774 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -2122,6 +2122,8 @@ Gem::Specification.new do |s| s.files += %w( third_party/abseil-cpp/absl/debugging/internal/demangle.h ) s.files += %w( third_party/abseil-cpp/absl/debugging/internal/elf_mem_image.cc ) s.files += %w( third_party/abseil-cpp/absl/debugging/internal/elf_mem_image.h ) + s.files += %w( third_party/abseil-cpp/absl/debugging/internal/examine_stack.cc ) + s.files += %w( third_party/abseil-cpp/absl/debugging/internal/examine_stack.h ) s.files += %w( third_party/abseil-cpp/absl/debugging/internal/stacktrace_aarch64-inl.inc ) s.files += %w( third_party/abseil-cpp/absl/debugging/internal/stacktrace_arm-inl.inc ) s.files += %w( third_party/abseil-cpp/absl/debugging/internal/stacktrace_config.h ) @@ -2179,6 +2181,40 @@ Gem::Specification.new do |s| s.files += %w( third_party/abseil-cpp/absl/hash/internal/hash.h ) s.files += %w( third_party/abseil-cpp/absl/hash/internal/low_level_hash.cc ) s.files += %w( third_party/abseil-cpp/absl/hash/internal/low_level_hash.h ) + s.files += %w( third_party/abseil-cpp/absl/log/absl_vlog_is_on.h ) + s.files += %w( third_party/abseil-cpp/absl/log/globals.cc ) + s.files += %w( third_party/abseil-cpp/absl/log/globals.h ) + s.files += %w( third_party/abseil-cpp/absl/log/internal/append_truncated.h ) + s.files += %w( third_party/abseil-cpp/absl/log/internal/conditions.cc ) + s.files += %w( third_party/abseil-cpp/absl/log/internal/conditions.h ) + s.files += %w( third_party/abseil-cpp/absl/log/internal/config.h ) + s.files += %w( third_party/abseil-cpp/absl/log/internal/fnmatch.cc ) + s.files += %w( third_party/abseil-cpp/absl/log/internal/fnmatch.h ) + s.files += %w( third_party/abseil-cpp/absl/log/internal/globals.cc ) + s.files += %w( third_party/abseil-cpp/absl/log/internal/globals.h ) + s.files += %w( third_party/abseil-cpp/absl/log/internal/log_format.cc ) + s.files += %w( third_party/abseil-cpp/absl/log/internal/log_format.h ) + s.files += %w( third_party/abseil-cpp/absl/log/internal/log_impl.h ) + s.files += %w( third_party/abseil-cpp/absl/log/internal/log_message.cc ) + s.files += %w( third_party/abseil-cpp/absl/log/internal/log_message.h ) + s.files += %w( third_party/abseil-cpp/absl/log/internal/log_sink_set.cc ) + s.files += %w( third_party/abseil-cpp/absl/log/internal/log_sink_set.h ) + s.files += %w( third_party/abseil-cpp/absl/log/internal/nullguard.cc ) + s.files += %w( third_party/abseil-cpp/absl/log/internal/nullguard.h ) + s.files += %w( third_party/abseil-cpp/absl/log/internal/nullstream.h ) + s.files += %w( third_party/abseil-cpp/absl/log/internal/proto.cc ) + s.files += %w( third_party/abseil-cpp/absl/log/internal/proto.h ) + s.files += %w( third_party/abseil-cpp/absl/log/internal/strip.h ) + s.files += %w( third_party/abseil-cpp/absl/log/internal/vlog_config.cc ) + s.files += %w( third_party/abseil-cpp/absl/log/internal/vlog_config.h ) + s.files += %w( third_party/abseil-cpp/absl/log/internal/voidify.h ) + s.files += %w( third_party/abseil-cpp/absl/log/log.h ) + s.files += %w( third_party/abseil-cpp/absl/log/log_entry.cc ) + s.files += %w( third_party/abseil-cpp/absl/log/log_entry.h ) + s.files += %w( third_party/abseil-cpp/absl/log/log_sink.cc ) + s.files += %w( third_party/abseil-cpp/absl/log/log_sink.h ) + s.files += %w( third_party/abseil-cpp/absl/log/log_sink_registry.h ) + s.files += %w( third_party/abseil-cpp/absl/log/vlog_is_on.h ) s.files += %w( third_party/abseil-cpp/absl/memory/memory.h ) s.files += %w( third_party/abseil-cpp/absl/meta/type_traits.h ) s.files += %w( third_party/abseil-cpp/absl/numeric/bits.h ) diff --git a/grpc.gyp b/grpc.gyp index df31e5f4f3b..eb70fc54080 100644 --- a/grpc.gyp +++ b/grpc.gyp @@ -194,9 +194,11 @@ 'dependencies': [ 'absl/base:base', 'absl/base:core_headers', + 'absl/base:log_severity', 'absl/flags:flag', 'absl/flags:marshalling', 'absl/functional:any_invocable', + 'absl/log:log', 'absl/memory:memory', 'absl/random:random', 'absl/status:status', diff --git a/package.xml b/package.xml index 84621351c14..c31c71f176b 100644 --- a/package.xml +++ b/package.xml @@ -2126,6 +2126,8 @@ + + @@ -2183,6 +2185,40 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/core/lib/config/config_vars.cc b/src/core/lib/config/config_vars.cc index 75b56d344c7..c13c6c6d81e 100644 --- a/src/core/lib/config/config_vars.cc +++ b/src/core/lib/config/config_vars.cc @@ -20,11 +20,8 @@ #include "src/core/lib/config/config_vars.h" -#include - #include "absl/flags/flag.h" #include "absl/strings/escaping.h" -#include "absl/strings/str_cat.h" #include "src/core/lib/config/load_config.h" @@ -78,6 +75,8 @@ ABSL_FLAG(absl::optional, grpc_not_use_system_ssl_roots, {}, "Disable loading system root certificates."); ABSL_FLAG(absl::optional, grpc_ssl_cipher_suites, {}, "A colon separated list of cipher suites to use with OpenSSL"); +ABSL_FLAG(absl::optional, grpc_absl_logging, {}, + "Use absl logging from within gpr_log."); namespace grpc_core { @@ -95,6 +94,8 @@ ConfigVars::ConfigVars(const Overrides& overrides) not_use_system_ssl_roots_(LoadConfig( FLAGS_grpc_not_use_system_ssl_roots, "GRPC_NOT_USE_SYSTEM_SSL_ROOTS", overrides.not_use_system_ssl_roots, false)), + absl_logging_(LoadConfig(FLAGS_grpc_absl_logging, "GRPC_ABSL_LOGGING", + overrides.absl_logging, false)), dns_resolver_(LoadConfig(FLAGS_grpc_dns_resolver, "GRPC_DNS_RESOLVER", overrides.dns_resolver, "")), verbosity_(LoadConfig(FLAGS_grpc_verbosity, "GRPC_VERBOSITY", @@ -146,7 +147,8 @@ std::string ConfigVars::ToString() const { "\"", ", default_ssl_roots_file_path: ", "\"", absl::CEscape(DefaultSslRootsFilePath()), "\"", ", not_use_system_ssl_roots: ", NotUseSystemSslRoots() ? "true" : "false", - ", ssl_cipher_suites: ", "\"", absl::CEscape(SslCipherSuites()), "\""); + ", ssl_cipher_suites: ", "\"", absl::CEscape(SslCipherSuites()), "\"", + ", absl_logging: ", AbslLogging() ? "true" : "false"); } } // namespace grpc_core diff --git a/src/core/lib/config/config_vars.h b/src/core/lib/config/config_vars.h index 0ca8ad9f530..c252b2809dc 100644 --- a/src/core/lib/config/config_vars.h +++ b/src/core/lib/config/config_vars.h @@ -38,6 +38,7 @@ class GPR_DLL ConfigVars { absl::optional enable_fork_support; absl::optional abort_on_leaks; absl::optional not_use_system_ssl_roots; + absl::optional absl_logging; absl::optional dns_resolver; absl::optional verbosity; absl::optional stacktrace_minloglevel; @@ -102,6 +103,8 @@ class GPR_DLL ConfigVars { bool NotUseSystemSslRoots() const { return not_use_system_ssl_roots_; } // A colon separated list of cipher suites to use with OpenSSL absl::string_view SslCipherSuites() const { return ssl_cipher_suites_; } + // Use absl logging from within gpr_log. + bool AbslLogging() const { return absl_logging_; } private: explicit ConfigVars(const Overrides& overrides); @@ -111,6 +114,7 @@ class GPR_DLL ConfigVars { bool enable_fork_support_; bool abort_on_leaks_; bool not_use_system_ssl_roots_; + bool absl_logging_; std::string dns_resolver_; std::string verbosity_; std::string stacktrace_minloglevel_; diff --git a/src/core/lib/config/config_vars.yaml b/src/core/lib/config/config_vars.yaml index 8ccd41c2d41..f3811327d6a 100644 --- a/src/core/lib/config/config_vars.yaml +++ b/src/core/lib/config/config_vars.yaml @@ -126,3 +126,8 @@ ECDHE-ECDSA-AES256-GCM-SHA384:\ ECDHE-RSA-AES128-GCM-SHA256:\ ECDHE-RSA-AES256-GCM-SHA384" +- name: absl_logging + type: bool + default: false + description: + Use absl logging from within gpr_log. diff --git a/src/core/lib/gpr/android/log.cc b/src/core/lib/gpr/android/log.cc index 92d17115361..34c705b8764 100644 --- a/src/core/lib/gpr/android/log.cc +++ b/src/core/lib/gpr/android/log.cc @@ -57,7 +57,7 @@ void gpr_log(const char* file, int line, gpr_log_severity severity, free(message); } -void gpr_default_log(gpr_log_func_args* args) { +void gpr_platform_log(gpr_log_func_args* args) { const char* final_slash; const char* display_file; char* output = NULL; diff --git a/src/core/lib/gpr/linux/log.cc b/src/core/lib/gpr/linux/log.cc index 7a597f3bb59..a24e28fa82c 100644 --- a/src/core/lib/gpr/linux/log.cc +++ b/src/core/lib/gpr/linux/log.cc @@ -70,7 +70,7 @@ void gpr_log(const char* file, int line, gpr_log_severity severity, free(message); } -void gpr_default_log(gpr_log_func_args* args) { +void gpr_platform_log(gpr_log_func_args* args) { const char* final_slash; const char* display_file; char time_buffer[64]; diff --git a/src/core/lib/gpr/log.cc b/src/core/lib/gpr/log.cc index 25c53f9a5c1..1755d99122d 100644 --- a/src/core/lib/gpr/log.cc +++ b/src/core/lib/gpr/log.cc @@ -18,6 +18,8 @@ #include +#include "absl/log/log.h" + #include #include @@ -40,6 +42,7 @@ static constexpr gpr_atm GPR_LOG_SEVERITY_UNSET = GPR_LOG_SEVERITY_ERROR + 10; static constexpr gpr_atm GPR_LOG_SEVERITY_NONE = GPR_LOG_SEVERITY_ERROR + 11; void gpr_default_log(gpr_log_func_args* args); +void gpr_platform_log(gpr_log_func_args* args); static gpr_atm g_log_func = reinterpret_cast(gpr_default_log); static gpr_atm g_min_severity_to_print = GPR_LOG_SEVERITY_UNSET; static gpr_atm g_min_severity_to_print_stacktrace = GPR_LOG_SEVERITY_UNSET; @@ -73,6 +76,29 @@ int gpr_should_log(gpr_log_severity severity) { : 0; } +void gpr_default_log(gpr_log_func_args* args) { + if (!grpc_core::ConfigVars::Get().AbslLogging()) { + gpr_platform_log(args); + return; + } + switch (args->severity) { + case GPR_LOG_SEVERITY_DEBUG: + // Log DEBUG messages as VLOG(2). + VLOG(2).AtLocation(args->file, args->line) << args->message; + return; + case GPR_LOG_SEVERITY_INFO: + LOG(INFO).AtLocation(args->file, args->line) << args->message; + return; + case GPR_LOG_SEVERITY_ERROR: + LOG(ERROR).AtLocation(args->file, args->line) << args->message; + return; + default: + LOG(ERROR) << __func__ << ": unknown gpr log severity(" << args->severity + << "), using ERROR"; + LOG(ERROR).AtLocation(args->file, args->line) << args->message; + } +} + int gpr_should_log_stacktrace(gpr_log_severity severity) { return static_cast(severity) >= gpr_atm_no_barrier_load(&g_min_severity_to_print_stacktrace) diff --git a/src/core/lib/gpr/posix/log.cc b/src/core/lib/gpr/posix/log.cc index 18088486118..4e933b7c4be 100644 --- a/src/core/lib/gpr/posix/log.cc +++ b/src/core/lib/gpr/posix/log.cc @@ -70,7 +70,7 @@ void gpr_log(const char* file, int line, gpr_log_severity severity, gpr_free(allocated); } -void gpr_default_log(gpr_log_func_args* args) { +void gpr_platform_log(gpr_log_func_args* args) { const char* final_slash; const char* display_file; char time_buffer[64]; diff --git a/src/core/lib/gpr/windows/log.cc b/src/core/lib/gpr/windows/log.cc index fe249e31c15..4dc48698140 100644 --- a/src/core/lib/gpr/windows/log.cc +++ b/src/core/lib/gpr/windows/log.cc @@ -73,7 +73,7 @@ void gpr_log(const char* file, int line, gpr_log_severity severity, } // Simple starter implementation -void gpr_default_log(gpr_log_func_args* args) { +void gpr_platform_log(gpr_log_func_args* args) { const char* final_slash; const char* display_file; char time_buffer[64]; diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index f3f4a909fd5..76daef457eb 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -888,6 +888,7 @@ CORE_SOURCE_FILES = [ 'third_party/abseil-cpp/absl/debugging/internal/address_is_readable.cc', 'third_party/abseil-cpp/absl/debugging/internal/demangle.cc', 'third_party/abseil-cpp/absl/debugging/internal/elf_mem_image.cc', + 'third_party/abseil-cpp/absl/debugging/internal/examine_stack.cc', 'third_party/abseil-cpp/absl/debugging/internal/vdso_support.cc', 'third_party/abseil-cpp/absl/debugging/stacktrace.cc', 'third_party/abseil-cpp/absl/debugging/symbolize.cc', @@ -902,6 +903,18 @@ CORE_SOURCE_FILES = [ 'third_party/abseil-cpp/absl/hash/internal/city.cc', 'third_party/abseil-cpp/absl/hash/internal/hash.cc', 'third_party/abseil-cpp/absl/hash/internal/low_level_hash.cc', + 'third_party/abseil-cpp/absl/log/globals.cc', + 'third_party/abseil-cpp/absl/log/internal/conditions.cc', + 'third_party/abseil-cpp/absl/log/internal/fnmatch.cc', + 'third_party/abseil-cpp/absl/log/internal/globals.cc', + 'third_party/abseil-cpp/absl/log/internal/log_format.cc', + 'third_party/abseil-cpp/absl/log/internal/log_message.cc', + 'third_party/abseil-cpp/absl/log/internal/log_sink_set.cc', + 'third_party/abseil-cpp/absl/log/internal/nullguard.cc', + 'third_party/abseil-cpp/absl/log/internal/proto.cc', + 'third_party/abseil-cpp/absl/log/internal/vlog_config.cc', + 'third_party/abseil-cpp/absl/log/log_entry.cc', + 'third_party/abseil-cpp/absl/log/log_sink.cc', 'third_party/abseil-cpp/absl/numeric/int128.cc', 'third_party/abseil-cpp/absl/profiling/internal/exponential_biased.cc', 'third_party/abseil-cpp/absl/random/discrete_distribution.cc', diff --git a/src/python/grpcio_observability/observability_lib_deps.py b/src/python/grpcio_observability/observability_lib_deps.py index eb117797788..fe161656970 100644 --- a/src/python/grpcio_observability/observability_lib_deps.py +++ b/src/python/grpcio_observability/observability_lib_deps.py @@ -89,6 +89,7 @@ CC_FILES=[ 'third_party/abseil-cpp/absl/debugging/internal/address_is_readable.cc', 'third_party/abseil-cpp/absl/debugging/internal/demangle.cc', 'third_party/abseil-cpp/absl/debugging/internal/elf_mem_image.cc', + 'third_party/abseil-cpp/absl/debugging/internal/examine_stack.cc', 'third_party/abseil-cpp/absl/debugging/internal/vdso_support.cc', 'third_party/abseil-cpp/absl/debugging/stacktrace.cc', 'third_party/abseil-cpp/absl/debugging/symbolize.cc', @@ -103,6 +104,18 @@ CC_FILES=[ 'third_party/abseil-cpp/absl/hash/internal/city.cc', 'third_party/abseil-cpp/absl/hash/internal/hash.cc', 'third_party/abseil-cpp/absl/hash/internal/low_level_hash.cc', + 'third_party/abseil-cpp/absl/log/globals.cc', + 'third_party/abseil-cpp/absl/log/internal/conditions.cc', + 'third_party/abseil-cpp/absl/log/internal/fnmatch.cc', + 'third_party/abseil-cpp/absl/log/internal/globals.cc', + 'third_party/abseil-cpp/absl/log/internal/log_format.cc', + 'third_party/abseil-cpp/absl/log/internal/log_message.cc', + 'third_party/abseil-cpp/absl/log/internal/log_sink_set.cc', + 'third_party/abseil-cpp/absl/log/internal/nullguard.cc', + 'third_party/abseil-cpp/absl/log/internal/proto.cc', + 'third_party/abseil-cpp/absl/log/internal/vlog_config.cc', + 'third_party/abseil-cpp/absl/log/log_entry.cc', + 'third_party/abseil-cpp/absl/log/log_sink.cc', 'third_party/abseil-cpp/absl/numeric/int128.cc', 'third_party/abseil-cpp/absl/profiling/internal/exponential_biased.cc', 'third_party/abseil-cpp/absl/random/discrete_distribution.cc', diff --git a/test/core/util/fuzz_config_vars.cc b/test/core/util/fuzz_config_vars.cc index 7fa9a4e51c7..7ec3b2382f1 100644 --- a/test/core/util/fuzz_config_vars.cc +++ b/test/core/util/fuzz_config_vars.cc @@ -18,10 +18,6 @@ #include "test/core/util/fuzz_config_vars.h" -#include - -#include "absl/types/optional.h" - #include "test/core/util/fuzz_config_vars_helpers.h" namespace grpc_core { From c910004328210668e0180847c35f9d2e82fa81dd Mon Sep 17 00:00:00 2001 From: Xuan Wang Date: Wed, 13 Mar 2024 22:44:21 -0700 Subject: [PATCH 42/52] [Python Stub] Add version check to stubs generated by grpcio_tools (#35906) The stubs generated by grpcio_tools should always be used with [the same or higher version of grpcio](https://github.com/grpc/grpc/blob/master/tools/distrib/python/grpcio_tools/setup.py#L313), this change will add a run time check for this requirement inside the generated stubs and therefor enforce this requirement. Please note for now we're just printing a warning for incorrect usage, we'll **change it to an error** soon. Example warning message: ``` /usr/local/google/home/xuanwn/workspace/misc/grpc/examples/python/helloworld/helloworld_pb2_grpc.py:21: RuntimeWarning: The grpc package installed is at version 1.60.1, but the generated code in helloworld_pb2_grpc.py depends on grpcio>=1.63.0.dev0. Please upgrade your grpc module to grpcio>=1.63.0.dev0 or downgrade your generated code using grpcio-tools<=1.60.1. This warning will become an error in 1.64.0, scheduled for release on May 14,2024. ``` Closes #35906 PiperOrigin-RevId: 615659471 --- examples/python/helloworld/helloworld_pb2.py | 2 +- .../python/helloworld/helloworld_pb2_grpc.py | 25 ++++++++ src/compiler/python_generator.cc | 62 ++++++++++++++++++- src/compiler/python_generator.h | 2 + src/python/grpcio/grpc/_utilities.py | 31 ++++++++++ src/python/grpcio_tests/tests/tests.json | 1 + .../grpcio_tests/tests/unit/BUILD.bazel | 1 + .../tests/unit/_utilities_test.py | 38 ++++++++++++ .../grpc_tools/grpc_version.py.template | 19 ++++++ tools/distrib/python/grpcio_tools/BUILD.bazel | 1 + tools/distrib/python/grpcio_tools/MANIFEST.in | 1 + .../grpc_tools/_protoc_compiler.pyx | 12 ++-- .../grpcio_tools/grpc_tools/grpc_version.py | 17 +++++ .../python/grpcio_tools/grpc_tools/main.cc | 15 +++-- .../python/grpcio_tools/grpc_tools/main.h | 5 +- 15 files changed, 220 insertions(+), 12 deletions(-) create mode 100644 src/python/grpcio_tests/tests/unit/_utilities_test.py create mode 100644 templates/tools/distrib/python/grpcio_tools/grpc_tools/grpc_version.py.template create mode 100644 tools/distrib/python/grpcio_tools/grpc_tools/grpc_version.py diff --git a/examples/python/helloworld/helloworld_pb2.py b/examples/python/helloworld/helloworld_pb2.py index 6ee31ad94a2..0e202a09116 100644 --- a/examples/python/helloworld/helloworld_pb2.py +++ b/examples/python/helloworld/helloworld_pb2.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: helloworld.proto -# Protobuf Python Version: 4.25.0 +# Protobuf Python Version: 4.25.1 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool diff --git a/examples/python/helloworld/helloworld_pb2_grpc.py b/examples/python/helloworld/helloworld_pb2_grpc.py index 68bcfef1756..48ad1004f9b 100644 --- a/examples/python/helloworld/helloworld_pb2_grpc.py +++ b/examples/python/helloworld/helloworld_pb2_grpc.py @@ -1,9 +1,34 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" import grpc +import warnings import helloworld_pb2 as helloworld__pb2 +GRPC_GENERATED_VERSION = '1.63.0.dev0' +GRPC_VERSION = grpc.__version__ +EXPECTED_ERROR_RELEASE = '1.65.0' +SCHEDULED_RELEASE_DATE = 'June 25, 2024' +_version_not_supported = False + +try: + from grpc._utilities import first_version_is_lower + _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) +except ImportError: + _version_not_supported = True + +if _version_not_supported: + warnings.warn( + f'The grpc package installed is at version {GRPC_VERSION},' + + f' but the generated code in helloworld_pb2_grpc.py depends on' + + f' grpcio>={GRPC_GENERATED_VERSION}.' + + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' + + f' This warning will become an error in {EXPECTED_ERROR_RELEASE},' + + f' scheduled for release on {SCHEDULED_RELEASE_DATE}.', + RuntimeWarning + ) + class GreeterStub(object): """The greeting service definition. diff --git a/src/compiler/python_generator.cc b/src/compiler/python_generator.cc index b7a3115bce0..53b6083c670 100644 --- a/src/compiler/python_generator.cc +++ b/src/compiler/python_generator.cc @@ -688,6 +688,9 @@ bool PrivateGenerator::PrintPreamble(grpc_generator::Printer* out) { StringMap var; var["Package"] = config.grpc_package_root; out->Print(var, "import $Package$\n"); + if (config.grpc_tools_version.size() > 0) { + out->Print(var, "import warnings\n"); + } if (generate_in_pb2_grpc) { out->Print("\n"); StringPairSet imports_set; @@ -732,6 +735,56 @@ bool PrivateGenerator::PrintPreamble(grpc_generator::Printer* out) { } out->Print(var, "$ImportStatement$ as $ModuleAlias$\n"); } + + // Checks if generate code is used with a supported grpcio version. + if (config.grpc_tools_version.size() > 0) { + var["ToolsVersion"] = config.grpc_tools_version; + out->Print(var, "\nGRPC_GENERATED_VERSION = '$ToolsVersion$'\n"); + out->Print("GRPC_VERSION = grpc.__version__\n"); + out->Print("EXPECTED_ERROR_RELEASE = '1.65.0'\n"); + out->Print("SCHEDULED_RELEASE_DATE = 'June 25, 2024'\n"); + out->Print("_version_not_supported = False\n\n"); + out->Print("try:\n"); + { + IndentScope raii_import_indent(out); + out->Print( + "from grpc._utilities import first_version_is_lower\n" + "_version_not_supported = first_version_is_lower(GRPC_VERSION, " + "GRPC_GENERATED_VERSION)\n"); + } + out->Print("except ImportError:\n"); + { + IndentScope raii_import_error_indent(out); + out->Print("_version_not_supported = True\n"); + } + out->Print("\nif _version_not_supported:\n"); + { + IndentScope raii_warning_indent(out); + out->Print("warnings.warn(\n"); + { + IndentScope raii_warning_string_indent(out); + std::string filename_without_ext = file->filename_without_ext(); + std::replace(filename_without_ext.begin(), filename_without_ext.end(), + '-', '_'); + var["Pb2GrpcFileName"] = filename_without_ext; + out->Print( + var, + "f'The grpc package installed is at version {GRPC_VERSION},'\n" + "+ f' but the generated code in $Pb2GrpcFileName$_pb2_grpc.py " + "depends on'\n" + "+ f' grpcio>={GRPC_GENERATED_VERSION}.'\n" + "+ f' Please upgrade your grpc module to " + "grpcio>={GRPC_GENERATED_VERSION}'\n" + "+ f' or downgrade your generated code using " + "grpcio-tools<={GRPC_VERSION}.'\n" + "+ f' This warning will become an error in " + "{EXPECTED_ERROR_RELEASE},'\n" + "+ f' scheduled for release on {SCHEDULED_RELEASE_DATE}.',\n" + "RuntimeWarning\n"); + } + out->Print(")\n"); + } + } } return true; } @@ -828,7 +881,14 @@ pair PrivateGenerator::GetGrpcServices() { GeneratorConfiguration::GeneratorConfiguration() : grpc_package_root("grpc"), beta_package_root("grpc.beta"), - import_prefix("") {} + import_prefix(""), + grpc_tools_version("") {} + +GeneratorConfiguration::GeneratorConfiguration(std::string version) + : grpc_package_root("grpc"), + beta_package_root("grpc.beta"), + import_prefix(""), + grpc_tools_version(version) {} PythonGrpcGenerator::PythonGrpcGenerator(const GeneratorConfiguration& config) : config_(config) {} diff --git a/src/compiler/python_generator.h b/src/compiler/python_generator.h index 12343ccde17..d65f310f64d 100644 --- a/src/compiler/python_generator.h +++ b/src/compiler/python_generator.h @@ -31,11 +31,13 @@ namespace grpc_python_generator { // that may be used internally at Google. struct GeneratorConfiguration { GeneratorConfiguration(); + GeneratorConfiguration(std::string version); std::string grpc_package_root; // TODO(https://github.com/grpc/grpc/issues/8622): Drop this. std::string beta_package_root; // TODO(https://github.com/protocolbuffers/protobuf/issues/888): Drop this. std::string import_prefix; + std::string grpc_tools_version; std::vector prefixes_to_filter; }; diff --git a/src/python/grpcio/grpc/_utilities.py b/src/python/grpcio/grpc/_utilities.py index 1ab8c4fa8f1..620cab3838c 100644 --- a/src/python/grpcio/grpc/_utilities.py +++ b/src/python/grpcio/grpc/_utilities.py @@ -189,3 +189,34 @@ def channel_ready_future(channel: grpc.Channel) -> _ChannelReadyFuture: ready_future = _ChannelReadyFuture(channel) ready_future.start() return ready_future + + +def first_version_is_lower(version1: str, version2: str) -> bool: + """ + Compares two versions in the format '1.60.1' or '1.60.1.dev0'. + + This method will be used in all stubs generated by grpcio-tools to check whether + the stub version is compatible with the runtime grpcio. + + Args: + version1: The first version string. + version2: The second version string. + + Returns: + True if version1 is lower, False otherwise. + """ + version1_list = version1.split(".") + version2_list = version2.split(".") + + try: + for i in range(3): + if int(version1_list[i]) < int(version2_list[i]): + return True + elif int(version1_list[i]) > int(version2_list[i]): + return False + except ValueError: + # Return false in case we can't convert version to int. + return False + + # The version without dev0 will be considered lower. + return len(version1_list) < len(version2_list) diff --git a/src/python/grpcio_tests/tests/tests.json b/src/python/grpcio_tests/tests/tests.json index 9c42ec7baa0..32406fd8833 100644 --- a/src/python/grpcio_tests/tests/tests.json +++ b/src/python/grpcio_tests/tests/tests.json @@ -84,6 +84,7 @@ "tests.unit._server_wait_for_termination_test.ServerWaitForTerminationTest", "tests.unit._session_cache_test.SSLSessionCacheTest", "tests.unit._signal_handling_test.SignalHandlingTest", + "tests.unit._utilities_test.UtilityTest", "tests.unit._version_test.VersionTest", "tests.unit._xds_credentials_test.XdsCredentialsTest", "tests.unit.beta._beta_features_test.BetaFeaturesTest", diff --git a/src/python/grpcio_tests/tests/unit/BUILD.bazel b/src/python/grpcio_tests/tests/unit/BUILD.bazel index 400db0fb55b..de04e87b41a 100644 --- a/src/python/grpcio_tests/tests/unit/BUILD.bazel +++ b/src/python/grpcio_tests/tests/unit/BUILD.bazel @@ -54,6 +54,7 @@ GRPCIO_TESTS_UNIT = [ "_server_shutdown_test.py", "_server_wait_for_termination_test.py", "_session_cache_test.py", + "_utilities_test.py", "_xds_credentials_test.py", ] diff --git a/src/python/grpcio_tests/tests/unit/_utilities_test.py b/src/python/grpcio_tests/tests/unit/_utilities_test.py new file mode 100644 index 00000000000..7cef5f87813 --- /dev/null +++ b/src/python/grpcio_tests/tests/unit/_utilities_test.py @@ -0,0 +1,38 @@ +# Copyright 2024 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Test of gRPC Python's utilities.""" + +import logging +import unittest + +from grpc._utilities import first_version_is_lower + + +class UtilityTest(unittest.TestCase): + def testVersionCheck(self): + self.assertTrue(first_version_is_lower("1.2.3", "1.2.4")) + self.assertTrue(first_version_is_lower("1.2.4", "10.2.3")) + self.assertTrue(first_version_is_lower("1.2.3", "1.2.3.dev0")) + self.assertFalse(first_version_is_lower("NOT_A_VERSION", "1.2.4")) + self.assertFalse(first_version_is_lower("1.2.3", "NOT_A_VERSION")) + self.assertFalse(first_version_is_lower("1.2.4", "1.2.3")) + self.assertFalse(first_version_is_lower("10.2.3", "1.2.4")) + self.assertFalse(first_version_is_lower("1.2.3dev0", "1.2.3")) + self.assertFalse(first_version_is_lower("1.2.3", "1.2.3dev0")) + self.assertFalse(first_version_is_lower("1.2.3.dev0", "1.2.3")) + + +if __name__ == "__main__": + logging.basicConfig() + unittest.main(verbosity=2) diff --git a/templates/tools/distrib/python/grpcio_tools/grpc_tools/grpc_version.py.template b/templates/tools/distrib/python/grpcio_tools/grpc_tools/grpc_version.py.template new file mode 100644 index 00000000000..de03b9bfbcc --- /dev/null +++ b/templates/tools/distrib/python/grpcio_tools/grpc_tools/grpc_version.py.template @@ -0,0 +1,19 @@ +%YAML 1.2 +--- | + # Copyright 2024 gRPC authors. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_tools/grpc_version.py.template`!!! + + VERSION = '${settings.python_version.pep440()}' diff --git a/tools/distrib/python/grpcio_tools/BUILD.bazel b/tools/distrib/python/grpcio_tools/BUILD.bazel index fac6bb3babd..ba24e779e7a 100644 --- a/tools/distrib/python/grpcio_tools/BUILD.bazel +++ b/tools/distrib/python/grpcio_tools/BUILD.bazel @@ -50,6 +50,7 @@ py_library( name = "grpc_tools", srcs = [ "grpc_tools/__init__.py", + "grpc_tools/grpc_version.py", "grpc_tools/protoc.py", ], data = [":well_known_protos"], diff --git a/tools/distrib/python/grpcio_tools/MANIFEST.in b/tools/distrib/python/grpcio_tools/MANIFEST.in index 4943751879e..acd97f57ef3 100644 --- a/tools/distrib/python/grpcio_tools/MANIFEST.in +++ b/tools/distrib/python/grpcio_tools/MANIFEST.in @@ -3,6 +3,7 @@ include grpc_version.py include protoc_deps.py include protoc_lib_deps.py include README.rst +include grpc_tools/grpc_version.py graft grpc_tools graft grpc_root graft third_party diff --git a/tools/distrib/python/grpcio_tools/grpc_tools/_protoc_compiler.pyx b/tools/distrib/python/grpcio_tools/grpc_tools/_protoc_compiler.pyx index 7b307e0475c..a69e6dc74e5 100644 --- a/tools/distrib/python/grpcio_tools/grpc_tools/_protoc_compiler.pyx +++ b/tools/distrib/python/grpcio_tools/grpc_tools/_protoc_compiler.pyx @@ -13,6 +13,7 @@ # limitations under the License. # distutils: language=c++ +cimport cpython from cython.operator cimport dereference from libc cimport stdlib from libcpp.string cimport string @@ -21,6 +22,8 @@ from libcpp.vector cimport vector import warnings +from grpc_tools import grpc_version + cdef extern from "grpc_tools/main.h" namespace "grpc_tools": cppclass cProtocError "::grpc_tools::ProtocError": @@ -35,13 +38,13 @@ cdef extern from "grpc_tools/main.h" namespace "grpc_tools": int column string message - int protoc_main(int argc, char *argv[]) + int protoc_main(int argc, char *argv[], char* version) int protoc_get_protos(char* protobuf_path, vector[string]* include_path, vector[pair[string, string]]* files_out, vector[cProtocError]* errors, vector[cProtocWarning]* wrnings) nogil except + - int protoc_get_services(char* protobuf_path, + int protoc_get_services(char* protobuf_path, char* version, vector[string]* include_path, vector[pair[string, string]]* files_out, vector[cProtocError]* errors, @@ -51,7 +54,7 @@ def run_main(list args not None): cdef char **argv = stdlib.malloc(len(args)*sizeof(char *)) for i in range(len(args)): argv[i] = args[i] - return protoc_main(len(args), argv) + return protoc_main(len(args), argv, grpc_version.VERSION.encode()) class ProtocError(Exception): def __init__(self, filename, line, column, message): @@ -129,7 +132,8 @@ def get_services(bytes protobuf_path, list include_paths): cdef vector[cProtocError] errors # NOTE: Abbreviated name used to avoid shadowing of the module name. cdef vector[cProtocWarning] wrnings - rc = protoc_get_services(protobuf_path, &c_include_paths, &files, &errors, &wrnings) + version = grpc_version.VERSION.encode() + rc = protoc_get_services(protobuf_path, version, &c_include_paths, &files, &errors, &wrnings) _handle_errors(rc, &errors, &wrnings, protobuf_path) return files diff --git a/tools/distrib/python/grpcio_tools/grpc_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_tools/grpc_version.py new file mode 100644 index 00000000000..1855d69da73 --- /dev/null +++ b/tools/distrib/python/grpcio_tools/grpc_tools/grpc_version.py @@ -0,0 +1,17 @@ +# Copyright 2024 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_tools/grpc_version.py.template`!!! + +VERSION = '1.63.0.dev0' diff --git a/tools/distrib/python/grpcio_tools/grpc_tools/main.cc b/tools/distrib/python/grpcio_tools/grpc_tools/main.cc index 9302cc75529..866b1e2866b 100644 --- a/tools/distrib/python/grpcio_tools/grpc_tools/main.cc +++ b/tools/distrib/python/grpcio_tools/grpc_tools/main.cc @@ -43,7 +43,7 @@ using ::google::protobuf::io::StringOutputStream; using ::google::protobuf::io::ZeroCopyOutputStream; namespace grpc_tools { -int protoc_main(int argc, char* argv[]) { +int protoc_main(int argc, char* argv[], char* version) { google::protobuf::compiler::CommandLineInterface cli; cli.AllowPlugins("protoc-"); @@ -57,8 +57,12 @@ int protoc_main(int argc, char* argv[]) { cli.RegisterGenerator("--pyi_out", &pyi_generator, "Generate Python pyi stub."); + // Get grpc_tools version + std::string grpc_tools_version = version; + // gRPC Python - grpc_python_generator::GeneratorConfiguration grpc_py_config; + grpc_python_generator::GeneratorConfiguration grpc_py_config( + grpc_tools_version); grpc_python_generator::PythonGrpcGenerator grpc_py_generator(grpc_py_config); cli.RegisterGenerator("--grpc_python_out", &grpc_py_generator, "Generate Python source file."); @@ -181,11 +185,14 @@ int protoc_get_protos( } int protoc_get_services( - char* protobuf_path, const std::vector* include_paths, + char* protobuf_path, char* version, + const std::vector* include_paths, std::vector>* files_out, std::vector<::grpc_tools::ProtocError>* errors, std::vector<::grpc_tools::ProtocWarning>* warnings) { - grpc_python_generator::GeneratorConfiguration grpc_py_config; + std::string grpc_tools_version = version; + grpc_python_generator::GeneratorConfiguration grpc_py_config( + grpc_tools_version); grpc_python_generator::PythonGrpcGenerator grpc_py_generator(grpc_py_config); return generate_code(&grpc_py_generator, protobuf_path, include_paths, files_out, errors, warnings); diff --git a/tools/distrib/python/grpcio_tools/grpc_tools/main.h b/tools/distrib/python/grpcio_tools/grpc_tools/main.h index 7fcce3068c6..847f598c49c 100644 --- a/tools/distrib/python/grpcio_tools/grpc_tools/main.h +++ b/tools/distrib/python/grpcio_tools/grpc_tools/main.h @@ -19,7 +19,7 @@ namespace grpc_tools { // We declare `protoc_main` here since we want access to it from Cython as an // extern but *without* triggering a dllimport declspec when on Windows. -int protoc_main(int argc, char* argv[]); +int protoc_main(int argc, char* argv[], char* version); struct ProtocError { std::string filename; @@ -40,7 +40,8 @@ int protoc_get_protos( std::vector* errors, std::vector* warnings); int protoc_get_services( - char* protobuf_path, const std::vector* include_paths, + char* protobuf_path, char* version, + const std::vector* include_paths, std::vector>* files_out, std::vector* errors, std::vector* warnings); } // end namespace grpc_tools From 24be69b9bd085736a3afedf518f1a8ced8383aa0 Mon Sep 17 00:00:00 2001 From: Xuan Wang Date: Thu, 14 Mar 2024 09:34:29 -0700 Subject: [PATCH 43/52] [Python O11y] Change public interface (#36094) Address comments from design review meeting, mainly: * Use `OpenTelemetryPlugin` as public API. * Use keyword args to build plugin. Closes #36094 PiperOrigin-RevId: 615807264 --- .../observability_greeter_client.py | 17 +- .../observability_greeter_server.py | 17 +- .../grpc_observability/BUILD.bazel | 1 - .../grpc_observability/__init__.py | 16 +- .../_open_telemetry_exporter.py | 38 --- .../_open_telemetry_observability.py | 177 +++++++++++-- .../_open_telemetry_plugin.py | 240 +++++------------- .../observability/_observability_api_test.py | 7 +- .../_open_telemetry_observability_test.py | 167 ++++++------ 9 files changed, 308 insertions(+), 372 deletions(-) delete mode 100644 src/python/grpcio_observability/grpc_observability/_open_telemetry_exporter.py diff --git a/examples/python/observability/observability_greeter_client.py b/examples/python/observability/observability_greeter_client.py index 5e0e024f93f..c31fc7e5779 100644 --- a/examples/python/observability/observability_greeter_client.py +++ b/examples/python/observability/observability_greeter_client.py @@ -16,7 +16,6 @@ from collections import defaultdict import logging import time -from typing import Optional import grpc import grpc_observability @@ -29,14 +28,6 @@ from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader OTEL_EXPORT_INTERVAL_S = 0.5 -class BaseOpenTelemetryPlugin(grpc_observability.OpenTelemetryPlugin): - def __init__(self, provider: MeterProvider): - self.provider = provider - - def get_meter_provider(self) -> Optional[MeterProvider]: - return self.provider - - def run(): all_metrics = defaultdict(list) otel_exporter = open_telemetry_exporter.OTelMetricExporter(all_metrics) @@ -45,9 +36,11 @@ def run(): export_interval_millis=OTEL_EXPORT_INTERVAL_S * 1000, ) provider = MeterProvider(metric_readers=[reader]) - otel_plugin = BaseOpenTelemetryPlugin(provider) - grpc_observability.start_open_telemetry_observability(plugins=[otel_plugin]) + otel_plugin = grpc_observability.OpenTelemetryPlugin( + meter_provider=provider + ) + otel_plugin.register_global() with grpc.insecure_channel(target="localhost:50051") as channel: stub = helloworld_pb2_grpc.GreeterStub(channel) @@ -56,7 +49,7 @@ def run(): print(f"Greeter client received: {response.message}") except grpc.RpcError as rpc_error: print("Call failed with code: ", rpc_error.code()) - grpc_observability.end_open_telemetry_observability() + otel_plugin.deregister_global() # Sleep to make sure all metrics are exported. time.sleep(5) diff --git a/examples/python/observability/observability_greeter_server.py b/examples/python/observability/observability_greeter_server.py index 542e68be42d..804cb44ab2a 100644 --- a/examples/python/observability/observability_greeter_server.py +++ b/examples/python/observability/observability_greeter_server.py @@ -17,7 +17,6 @@ from collections import defaultdict from concurrent import futures import logging import time -from typing import Optional import grpc import grpc_observability @@ -31,14 +30,6 @@ _OTEL_EXPORT_INTERVAL_S = 0.5 _SERVER_PORT = "50051" -class BaseOpenTelemetryPlugin(grpc_observability.OpenTelemetryPlugin): - def __init__(self, provider: MeterProvider): - self.provider = provider - - def get_meter_provider(self) -> Optional[MeterProvider]: - return self.provider - - class Greeter(helloworld_pb2_grpc.GreeterServicer): def SayHello(self, request, context): message = request.name @@ -55,9 +46,11 @@ def serve(): export_interval_millis=_OTEL_EXPORT_INTERVAL_S * 1000, ) provider = MeterProvider(metric_readers=[reader]) - otel_plugin = BaseOpenTelemetryPlugin(provider) - grpc_observability.start_open_telemetry_observability(plugins=[otel_plugin]) + otel_plugin = grpc_observability.OpenTelemetryPlugin( + meter_provider=provider + ) + otel_plugin.register_global() server = grpc.server( thread_pool=futures.ThreadPoolExecutor(max_workers=10), @@ -74,7 +67,7 @@ def serve(): print(metric) server.stop(0) - grpc_observability.end_open_telemetry_observability() + otel_plugin.deregister_global() if __name__ == "__main__": diff --git a/src/python/grpcio_observability/grpc_observability/BUILD.bazel b/src/python/grpcio_observability/grpc_observability/BUILD.bazel index afb35e43c3f..1a760338114 100644 --- a/src/python/grpcio_observability/grpc_observability/BUILD.bazel +++ b/src/python/grpcio_observability/grpc_observability/BUILD.bazel @@ -58,7 +58,6 @@ pyx_library( py_library( name = "_opentelemetry_observability", srcs = [ - "_open_telemetry_exporter.py", "_open_telemetry_measures.py", "_open_telemetry_observability.py", "_open_telemetry_plugin.py", diff --git a/src/python/grpcio_observability/grpc_observability/__init__.py b/src/python/grpcio_observability/grpc_observability/__init__.py index f4a7ef453e9..6b228eb3846 100644 --- a/src/python/grpcio_observability/grpc_observability/__init__.py +++ b/src/python/grpcio_observability/grpc_observability/__init__.py @@ -12,20 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from grpc_observability._open_telemetry_observability import ( - OpenTelemetryObservability, -) -from grpc_observability._open_telemetry_observability import ( - end_open_telemetry_observability, -) -from grpc_observability._open_telemetry_observability import ( - start_open_telemetry_observability, -) from grpc_observability._open_telemetry_plugin import OpenTelemetryPlugin -__all__ = ( - "OpenTelemetryObservability", - "OpenTelemetryPlugin", - "start_open_telemetry_observability", - "end_open_telemetry_observability", -) +__all__ = ("OpenTelemetryPlugin",) diff --git a/src/python/grpcio_observability/grpc_observability/_open_telemetry_exporter.py b/src/python/grpcio_observability/grpc_observability/_open_telemetry_exporter.py deleted file mode 100644 index be2738a8595..00000000000 --- a/src/python/grpcio_observability/grpc_observability/_open_telemetry_exporter.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2023 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Iterable, List - -from grpc_observability import _observability # pytype: disable=pyi-error -from grpc_observability._open_telemetry_plugin import _OpenTelemetryPlugin - - -class _OpenTelemetryExporterDelegator(_observability.Exporter): - _plugins: Iterable[_OpenTelemetryPlugin] - - def __init__(self, plugins: Iterable[_OpenTelemetryPlugin]): - self._plugins = plugins - - def export_stats_data( - self, stats_data: List[_observability.StatsData] - ) -> None: - # Records stats data to MeterProvider. - for data in stats_data: - for plugin in self._plugins: - plugin.maybe_record_stats_data(data) - - def export_tracing_data( - self, tracing_data: List[_observability.TracingData] - ) -> None: - pass diff --git a/src/python/grpcio_observability/grpc_observability/_open_telemetry_observability.py b/src/python/grpcio_observability/grpc_observability/_open_telemetry_observability.py index 74f40b66a1f..7846c5568f5 100644 --- a/src/python/grpcio_observability/grpc_observability/_open_telemetry_observability.py +++ b/src/python/grpcio_observability/grpc_observability/_open_telemetry_observability.py @@ -15,17 +15,19 @@ import logging import threading import time -from typing import Any, Iterable, Optional +from typing import Any, Dict, Iterable, List, Optional, Union import grpc # pytype: disable=pyi-error from grpc_observability import _cyobservability -from grpc_observability._open_telemetry_exporter import ( - _OpenTelemetryExporterDelegator, -) -from grpc_observability._open_telemetry_plugin import OpenTelemetryPlugin -from grpc_observability._open_telemetry_plugin import _OpenTelemetryPlugin +from grpc_observability import _observability +from grpc_observability import _open_telemetry_measures +from grpc_observability._cyobservability import MetricsName +from grpc_observability._observability import StatsData +from opentelemetry.metrics import Counter +from opentelemetry.metrics import Histogram +from opentelemetry.metrics import Meter _LOGGER = logging.getLogger(__name__) @@ -34,6 +36,13 @@ ServerCallTracerFactoryCapsule = ( Any # it appears only once in the function signature ) grpc_observability = Any # grpc_observability.py imports this module. +OpenTelemetryPlugin = Any # _open_telemetry_plugin.py imports this module. + +GRPC_METHOD_LABEL = "grpc.method" +GRPC_TARGET_LABEL = "grpc.target" +GRPC_OTHER_LABEL_VALUE = "other" +_observability_lock: threading.RLock = threading.RLock() +_OPEN_TELEMETRY_OBSERVABILITY: Optional["OpenTelemetryObservability"] = None GRPC_STATUS_CODE_TO_STRING = { grpc.StatusCode.OK: "OK", @@ -55,13 +64,125 @@ GRPC_STATUS_CODE_TO_STRING = { grpc.StatusCode.DATA_LOSS: "DATA_LOSS", } -_observability_lock: threading.RLock = threading.RLock() -_OPEN_TELEMETRY_OBSERVABILITY: Optional["OpenTelemetryObservability"] = None + +class _OpenTelemetryPlugin: + _plugin: OpenTelemetryPlugin + _metric_to_recorder: Dict[MetricsName, Union[Counter, Histogram]] + + def __init__(self, plugin: OpenTelemetryPlugin): + self._plugin = plugin + self._metric_to_recorder = dict() + + meter_provider = self._plugin.meter_provider + if meter_provider: + meter = meter_provider.get_meter("grpc-python", grpc.__version__) + enabled_metrics = _open_telemetry_measures.base_metrics() + self._metric_to_recorder = self._register_metrics( + meter, enabled_metrics + ) + + def _should_record(self, stats_data: StatsData) -> bool: + # Decide if this plugin should record the stats_data. + return stats_data.name in self._metric_to_recorder.keys() + + def _record_stats_data(self, stats_data: StatsData) -> None: + recorder = self._metric_to_recorder[stats_data.name] + + target = stats_data.labels.get(GRPC_TARGET_LABEL, "") + if not self._plugin.target_attribute_filter(target): + # Filter target name. + stats_data.labels[GRPC_TARGET_LABEL] = GRPC_OTHER_LABEL_VALUE + + method = stats_data.labels.get(GRPC_METHOD_LABEL, "") + if not self._plugin.generic_method_attribute_filter(method): + # Filter method name. + stats_data.labels[GRPC_METHOD_LABEL] = GRPC_OTHER_LABEL_VALUE + + value = 0 + if stats_data.measure_double: + value = stats_data.value_float + else: + value = stats_data.value_int + if isinstance(recorder, Counter): + recorder.add(value, attributes=stats_data.labels) + elif isinstance(recorder, Histogram): + recorder.record(value, attributes=stats_data.labels) + + # pylint: disable=no-self-use + def maybe_record_stats_data(self, stats_data: List[StatsData]) -> None: + # Records stats data to MeterProvider. + if self._should_record(stats_data): + self._record_stats_data(stats_data) + + def _register_metrics( + self, meter: Meter, metrics: List[_open_telemetry_measures.Metric] + ) -> Dict[MetricsName, Union[Counter, Histogram]]: + metric_to_recorder_map = {} + recorder = None + for metric in metrics: + if metric == _open_telemetry_measures.CLIENT_ATTEMPT_STARTED: + recorder = meter.create_counter( + name=metric.name, + unit=metric.unit, + description=metric.description, + ) + elif metric == _open_telemetry_measures.CLIENT_ATTEMPT_DURATION: + recorder = meter.create_histogram( + name=metric.name, + unit=metric.unit, + description=metric.description, + ) + elif metric == _open_telemetry_measures.CLIENT_RPC_DURATION: + recorder = meter.create_histogram( + name=metric.name, + unit=metric.unit, + description=metric.description, + ) + elif metric == _open_telemetry_measures.CLIENT_ATTEMPT_SEND_BYTES: + recorder = meter.create_histogram( + name=metric.name, + unit=metric.unit, + description=metric.description, + ) + elif ( + metric == _open_telemetry_measures.CLIENT_ATTEMPT_RECEIVED_BYTES + ): + recorder = meter.create_histogram( + name=metric.name, + unit=metric.unit, + description=metric.description, + ) + elif metric == _open_telemetry_measures.SERVER_STARTED_RPCS: + recorder = meter.create_counter( + name=metric.name, + unit=metric.unit, + description=metric.description, + ) + elif metric == _open_telemetry_measures.SERVER_RPC_DURATION: + recorder = meter.create_histogram( + name=metric.name, + unit=metric.unit, + description=metric.description, + ) + elif metric == _open_telemetry_measures.SERVER_RPC_SEND_BYTES: + recorder = meter.create_histogram( + name=metric.name, + unit=metric.unit, + description=metric.description, + ) + elif metric == _open_telemetry_measures.SERVER_RPC_RECEIVED_BYTES: + recorder = meter.create_histogram( + name=metric.name, + unit=metric.unit, + description=metric.description, + ) + metric_to_recorder_map[metric.cyname] = recorder + return metric_to_recorder_map def start_open_telemetry_observability( *, - plugins: Optional[Iterable[OpenTelemetryPlugin]] = None, + plugins: Iterable[_OpenTelemetryPlugin], ) -> None: _start_open_telemetry_observability( OpenTelemetryObservability(plugins=plugins) @@ -72,6 +193,26 @@ def end_open_telemetry_observability() -> None: _end_open_telemetry_observability() +class _OpenTelemetryExporterDelegator(_observability.Exporter): + _plugins: Iterable[_OpenTelemetryPlugin] + + def __init__(self, plugins: Iterable[_OpenTelemetryPlugin]): + self._plugins = plugins + + def export_stats_data( + self, stats_data: List[_observability.StatsData] + ) -> None: + # Records stats data to MeterProvider. + for data in stats_data: + for plugin in self._plugins: + plugin.maybe_record_stats_data(data) + + def export_tracing_data( + self, tracing_data: List[_observability.TracingData] + ) -> None: + pass + + # pylint: disable=no-self-use class OpenTelemetryObservability(grpc._observability.ObservabilityPlugin): """OpenTelemetry based plugin implementation. @@ -79,7 +220,7 @@ class OpenTelemetryObservability(grpc._observability.ObservabilityPlugin): This is class is part of an EXPERIMENTAL API. Args: - plugin: OpenTelemetryPlugin to enable. + plugin: _OpenTelemetryPlugin to enable. """ exporter: "grpc_observability.Exporter" @@ -87,21 +228,9 @@ class OpenTelemetryObservability(grpc._observability.ObservabilityPlugin): def __init__( self, *, - plugins: Optional[Iterable[OpenTelemetryPlugin]] = None, + plugins: Optional[Iterable[_OpenTelemetryPlugin]], ): - _plugins = [] - if plugins: - for plugin in plugins: - _plugins.append(_OpenTelemetryPlugin(plugin)) - - self.exporter = _OpenTelemetryExporterDelegator(_plugins) - - def __enter__(self): - _start_open_telemetry_observability(self) - return self - - def __exit__(self, exc_type, exc_val, exc_tb) -> None: - _end_open_telemetry_observability() + self.exporter = _OpenTelemetryExporterDelegator(plugins) def observability_init(self): try: diff --git a/src/python/grpcio_observability/grpc_observability/_open_telemetry_plugin.py b/src/python/grpcio_observability/grpc_observability/_open_telemetry_plugin.py index fd782bdef47..c2da412b95e 100644 --- a/src/python/grpcio_observability/grpc_observability/_open_telemetry_plugin.py +++ b/src/python/grpcio_observability/grpc_observability/_open_telemetry_plugin.py @@ -13,22 +13,12 @@ # limitations under the License. import abc -from typing import Dict, Iterable, List, Optional, Union +from typing import Callable, Dict, Iterable, List, Optional # pytype: disable=pyi-error -import grpc -from grpc_observability import _open_telemetry_measures -from grpc_observability._cyobservability import MetricsName -from grpc_observability._observability import StatsData -from opentelemetry.metrics import Counter -from opentelemetry.metrics import Histogram -from opentelemetry.metrics import Meter +from grpc_observability import _open_telemetry_observability from opentelemetry.metrics import MeterProvider -GRPC_METHOD_LABEL = "grpc.method" -GRPC_TARGET_LABEL = "grpc.target" -GRPC_OTHER_LABEL_VALUE = "other" - class OpenTelemetryLabelInjector(abc.ABC): """ @@ -90,184 +80,80 @@ class OpenTelemetryPluginOption(abc.ABC): # pylint: disable=no-self-use class OpenTelemetryPlugin: - """Describes a Plugin for OpenTelemetry observability. + """Describes a Plugin for OpenTelemetry observability.""" - This is class is part of an EXPERIMENTAL API. - """ + plugin_options: Iterable[OpenTelemetryPluginOption] + meter_provider: Optional[MeterProvider] + target_attribute_filter: Callable[[str], bool] + generic_method_attribute_filter: Callable[[str], bool] + _plugin: _open_telemetry_observability._OpenTelemetryPlugin - def get_plugin_options( + def __init__( self, - ) -> Iterable[OpenTelemetryPluginOption]: - """ - This function will be used to get plugin options which are enabled for - this OpenTelemetryPlugin instance. - - Returns: - An Iterable of class OpenTelemetryPluginOption which will be enabled for - this OpenTelemetryPlugin. - """ - return [] - - def get_meter_provider(self) -> Optional[MeterProvider]: - """ - This function will be used to get the MeterProvider for this OpenTelemetryPlugin - instance. - - Returns: - A MeterProvider which will be used to collect telemetry data, or None which - means no metrics will be collected. - """ - return None - - def target_attribute_filter( - self, target: str # pylint: disable=unused-argument - ) -> bool: + *, + plugin_options: Iterable[OpenTelemetryPluginOption] = [], + meter_provider: Optional[MeterProvider] = None, + target_attribute_filter: Optional[Callable[[str], bool]] = None, + generic_method_attribute_filter: Optional[Callable[[str], bool]] = None, + ): """ - Once overridden, this will be called per channel to decide whether to record the - target attribute on client or to replace it with "other". + Args: + plugin_options: An Iterable of OpenTelemetryPluginOption which will be + enabled for this OpenTelemetryPlugin. + meter_provider: A MeterProvider which will be used to collect telemetry data, + or None which means no metrics will be collected. + target_attribute_filter: Once provided, this will be called per channel to decide + whether to record the target attribute on client or to replace it with "other". This helps reduce the cardinality on metrics in cases where many channels are created with different targets in the same binary (which might happen for example, if the channel target string uses IP addresses directly). - - Args: - target: The target for the RPC. - - Returns: - bool: True means the original target string will be used, False means target string - will be replaced with "other". + Return True means the original target string will be used, False means target string + will be replaced with "other". + generic_method_attribute_filter: Once provided, this will be called with a generic + method type to decide whether to record the method name or to replace it with + "other". Note that pre-registered methods will always be recorded no matter what + this function returns. + Return True means the original method name will be used, False means method name will + be replaced with "other". """ - return True + self.plugin_options = plugin_options + self.meter_provider = meter_provider + if target_attribute_filter: + self.target_attribute_filter = target_attribute_filter + else: + self.target_attribute_filter = lambda target: True + if generic_method_attribute_filter: + self.generic_method_attribute_filter = ( + generic_method_attribute_filter + ) + else: + self.generic_method_attribute_filter = lambda method: False + self._plugin = _open_telemetry_observability._OpenTelemetryPlugin(self) - def generic_method_attribute_filter( - self, method: str # pylint: disable=unused-argument - ) -> bool: + def register_global(self) -> None: """ - Once overridden, this will be called with a generic method type to decide whether to - record the method name or to replace it with "other". - - Note that pre-registered methods will always be recorded no matter what this - function returns. - - Args: - method: The method name for the RPC. + Registers a global plugin that acts on all channels and servers running on the process. - Returns: - bool: True means the original method name will be used, False means method name - will be replaced with "other". + Raises: + RuntimeError: If a global plugin was already registered. """ - return False - - -class _OpenTelemetryPlugin: - _plugin: OpenTelemetryPlugin - _metric_to_recorder: Dict[MetricsName, Union[Counter, Histogram]] - - def __init__(self, plugin: OpenTelemetryPlugin): - self._plugin = plugin - self._metric_to_recorder = dict() + _open_telemetry_observability.start_open_telemetry_observability( + plugins=[self._plugin] + ) - meter_provider = self._plugin.get_meter_provider() - if meter_provider: - meter = meter_provider.get_meter("grpc-python", grpc.__version__) - enabled_metrics = _open_telemetry_measures.base_metrics() - self._metric_to_recorder = self._register_metrics( - meter, enabled_metrics - ) - - def _should_record(self, stats_data: StatsData) -> bool: - # Decide if this plugin should record the stats_data. - return stats_data.name in self._metric_to_recorder.keys() - - def _record_stats_data(self, stats_data: StatsData) -> None: - recorder = self._metric_to_recorder[stats_data.name] - - target = stats_data.labels.get(GRPC_TARGET_LABEL, "") - if not self._plugin.target_attribute_filter(target): - # Filter target name. - stats_data.labels[GRPC_TARGET_LABEL] = GRPC_OTHER_LABEL_VALUE - - method = stats_data.labels.get(GRPC_METHOD_LABEL, "") - if not self._plugin.generic_method_attribute_filter(method): - # Filter method name. - stats_data.labels[GRPC_METHOD_LABEL] = GRPC_OTHER_LABEL_VALUE + def deregister_global(self) -> None: + """ + De-register the global plugin that acts on all channels and servers running on the process. - value = 0 - if stats_data.measure_double: - value = stats_data.value_float - else: - value = stats_data.value_int - if isinstance(recorder, Counter): - recorder.add(value, attributes=stats_data.labels) - elif isinstance(recorder, Histogram): - recorder.record(value, attributes=stats_data.labels) + Raises: + RuntimeError: If no global plugin was registered. + """ + _open_telemetry_observability.end_open_telemetry_observability() - # pylint: disable=no-self-use - def maybe_record_stats_data(self, stats_data: List[StatsData]) -> None: - # Records stats data to MeterProvider. - if self._should_record(stats_data): - self._record_stats_data(stats_data) + def __enter__(self) -> None: + _open_telemetry_observability.start_open_telemetry_observability( + plugins=[self._plugin] + ) - def _register_metrics( - self, meter: Meter, metrics: List[_open_telemetry_measures.Metric] - ) -> Dict[MetricsName, Union[Counter, Histogram]]: - metric_to_recorder_map = {} - recorder = None - for metric in metrics: - if metric == _open_telemetry_measures.CLIENT_ATTEMPT_STARTED: - recorder = meter.create_counter( - name=metric.name, - unit=metric.unit, - description=metric.description, - ) - elif metric == _open_telemetry_measures.CLIENT_ATTEMPT_DURATION: - recorder = meter.create_histogram( - name=metric.name, - unit=metric.unit, - description=metric.description, - ) - elif metric == _open_telemetry_measures.CLIENT_RPC_DURATION: - recorder = meter.create_histogram( - name=metric.name, - unit=metric.unit, - description=metric.description, - ) - elif metric == _open_telemetry_measures.CLIENT_ATTEMPT_SEND_BYTES: - recorder = meter.create_histogram( - name=metric.name, - unit=metric.unit, - description=metric.description, - ) - elif ( - metric == _open_telemetry_measures.CLIENT_ATTEMPT_RECEIVED_BYTES - ): - recorder = meter.create_histogram( - name=metric.name, - unit=metric.unit, - description=metric.description, - ) - elif metric == _open_telemetry_measures.SERVER_STARTED_RPCS: - recorder = meter.create_counter( - name=metric.name, - unit=metric.unit, - description=metric.description, - ) - elif metric == _open_telemetry_measures.SERVER_RPC_DURATION: - recorder = meter.create_histogram( - name=metric.name, - unit=metric.unit, - description=metric.description, - ) - elif metric == _open_telemetry_measures.SERVER_RPC_SEND_BYTES: - recorder = meter.create_histogram( - name=metric.name, - unit=metric.unit, - description=metric.description, - ) - elif metric == _open_telemetry_measures.SERVER_RPC_RECEIVED_BYTES: - recorder = meter.create_histogram( - name=metric.name, - unit=metric.unit, - description=metric.description, - ) - metric_to_recorder_map[metric.cyname] = recorder - return metric_to_recorder_map + def __exit__(self, exc_type, exc_val, exc_tb) -> None: + _open_telemetry_observability.end_open_telemetry_observability() diff --git a/src/python/grpcio_tests/tests/observability/_observability_api_test.py b/src/python/grpcio_tests/tests/observability/_observability_api_test.py index e3f92338de5..9e3c3360279 100644 --- a/src/python/grpcio_tests/tests/observability/_observability_api_test.py +++ b/src/python/grpcio_tests/tests/observability/_observability_api_test.py @@ -21,12 +21,7 @@ from tests.observability import _from_observability_import_star class AllTest(unittest.TestCase): def testAll(self): - expected_observability_code_elements = ( - "OpenTelemetryObservability", - "OpenTelemetryPlugin", - "start_open_telemetry_observability", - "end_open_telemetry_observability", - ) + expected_observability_code_elements = ("OpenTelemetryPlugin",) self.assertCountEqual( expected_observability_code_elements, diff --git a/src/python/grpcio_tests/tests/observability/_open_telemetry_observability_test.py b/src/python/grpcio_tests/tests/observability/_open_telemetry_observability_test.py index 7031965b49f..2a0031af34a 100644 --- a/src/python/grpcio_tests/tests/observability/_open_telemetry_observability_test.py +++ b/src/python/grpcio_tests/tests/observability/_open_telemetry_observability_test.py @@ -20,14 +20,15 @@ import sys import time from typing import Any, Callable, Dict, List, Optional, Set import unittest -from unittest.mock import patch import grpc import grpc_observability from grpc_observability import _open_telemetry_measures -from grpc_observability._open_telemetry_plugin import GRPC_METHOD_LABEL -from grpc_observability._open_telemetry_plugin import GRPC_OTHER_LABEL_VALUE -from grpc_observability._open_telemetry_plugin import GRPC_TARGET_LABEL +from grpc_observability._open_telemetry_observability import ( + GRPC_OTHER_LABEL_VALUE, +) +from grpc_observability._open_telemetry_observability import GRPC_METHOD_LABEL +from grpc_observability._open_telemetry_observability import GRPC_TARGET_LABEL from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import AggregationTemporality from opentelemetry.sdk.metrics.export import MetricExportResult @@ -94,14 +95,6 @@ class OTelMetricExporter(MetricExporter): ) -class BaseTestOpenTelemetryPlugin(grpc_observability.OpenTelemetryPlugin): - def __init__(self, provider: MeterProvider): - self.provider = provider - - def get_meter_provider(self) -> Optional[MeterProvider]: - return self.provider - - class _ClientUnaryUnaryInterceptor(grpc.UnaryUnaryClientInterceptor): def intercept_unary_unary( self, continuation, client_call_details, request_or_iterator @@ -136,9 +129,8 @@ class OpenTelemetryObservabilityTest(unittest.TestCase): self._server.stop(0) def testRecordUnaryUnaryUseContextManager(self): - otel_plugin = BaseTestOpenTelemetryPlugin(self._provider) - with grpc_observability.OpenTelemetryObservability( - plugins=[otel_plugin] + with grpc_observability.OpenTelemetryPlugin( + meter_provider=self._provider ): server, port = _test_server.start_server() self._server = server @@ -148,36 +140,42 @@ class OpenTelemetryObservabilityTest(unittest.TestCase): self._validate_all_metrics_names(self.all_metrics) def testRecordUnaryUnaryUseGlobalInit(self): - otel_plugin = BaseTestOpenTelemetryPlugin(self._provider) - - grpc_observability.start_open_telemetry_observability( - plugins=[otel_plugin] + otel_plugin = grpc_observability.OpenTelemetryPlugin( + meter_provider=self._provider ) + otel_plugin.register_global() + server, port = _test_server.start_server() self._server = server _test_server.unary_unary_call(port=port) self._validate_metrics_exist(self.all_metrics) self._validate_all_metrics_names(self.all_metrics) - grpc_observability.end_open_telemetry_observability() + otel_plugin.deregister_global() def testCallGlobalInitThrowErrorWhenGlobalCalled(self): - grpc_observability.start_open_telemetry_observability(plugins=[]) + otel_plugin = grpc_observability.OpenTelemetryPlugin( + meter_provider=self._provider + ) + otel_plugin.register_global() try: - grpc_observability.start_open_telemetry_observability(plugins=[]) + otel_plugin.register_global() except RuntimeError as exp: self.assertIn( "gPRC Python observability was already initialized", str(exp) ) - grpc_observability.end_open_telemetry_observability() + otel_plugin.deregister_global() def testCallGlobalInitThrowErrorWhenContextManagerCalled(self): - with grpc_observability.OpenTelemetryObservability(plugins=[]): + with grpc_observability.OpenTelemetryPlugin( + meter_provider=self._provider + ): try: - grpc_observability.start_open_telemetry_observability( - plugins=[] + otel_plugin = grpc_observability.OpenTelemetryPlugin( + meter_provider=self._provider ) + otel_plugin.register_global() except RuntimeError as exp: self.assertIn( "gPRC Python observability was already initialized", @@ -185,20 +183,29 @@ class OpenTelemetryObservabilityTest(unittest.TestCase): ) def testCallContextManagerThrowErrorWhenGlobalInitCalled(self): - grpc_observability.start_open_telemetry_observability(plugins=[]) + otel_plugin = grpc_observability.OpenTelemetryPlugin( + meter_provider=self._provider + ) + otel_plugin.register_global() try: - with grpc_observability.OpenTelemetryObservability(plugins=[]): + with grpc_observability.OpenTelemetryPlugin( + meter_provider=self._provider + ): pass except RuntimeError as exp: self.assertIn( "gPRC Python observability was already initialized", str(exp) ) - grpc_observability.end_open_telemetry_observability() + otel_plugin.deregister_global() def testContextManagerThrowErrorWhenContextManagerCalled(self): - with grpc_observability.OpenTelemetryObservability(plugins=[]): + with grpc_observability.OpenTelemetryPlugin( + meter_provider=self._provider + ): try: - with grpc_observability.OpenTelemetryObservability(plugins=[]): + with grpc_observability.OpenTelemetryPlugin( + meter_provider=self._provider + ): pass except RuntimeError as exp: self.assertIn( @@ -207,23 +214,32 @@ class OpenTelemetryObservabilityTest(unittest.TestCase): ) def testNoErrorCallGlobalInitThenContextManager(self): - grpc_observability.start_open_telemetry_observability(plugins=[]) - grpc_observability.end_open_telemetry_observability() + otel_plugin = grpc_observability.OpenTelemetryPlugin( + meter_provider=self._provider + ) + otel_plugin.register_global() + otel_plugin.deregister_global() - with grpc_observability.OpenTelemetryObservability(plugins=[]): + with grpc_observability.OpenTelemetryPlugin( + meter_provider=self._provider + ): pass def testNoErrorCallContextManagerThenGlobalInit(self): - with grpc_observability.OpenTelemetryObservability(plugins=[]): + with grpc_observability.OpenTelemetryPlugin( + meter_provider=self._provider + ): pass - grpc_observability.start_open_telemetry_observability(plugins=[]) - grpc_observability.end_open_telemetry_observability() + otel_plugin = grpc_observability.OpenTelemetryPlugin( + meter_provider=self._provider + ) + otel_plugin.register_global() + otel_plugin.deregister_global() def testRecordUnaryUnaryWithClientInterceptor(self): interceptor = _ClientUnaryUnaryInterceptor() - otel_plugin = BaseTestOpenTelemetryPlugin(self._provider) - with grpc_observability.OpenTelemetryObservability( - plugins=[otel_plugin] + with grpc_observability.OpenTelemetryPlugin( + meter_provider=self._provider ): server, port = _test_server.start_server() self._server = server @@ -236,9 +252,8 @@ class OpenTelemetryObservabilityTest(unittest.TestCase): def testRecordUnaryUnaryWithServerInterceptor(self): interceptor = _ServerInterceptor() - otel_plugin = BaseTestOpenTelemetryPlugin(self._provider) - with grpc_observability.OpenTelemetryObservability( - plugins=[otel_plugin] + with grpc_observability.OpenTelemetryPlugin( + meter_provider=self._provider ): server, port = _test_server.start_server(interceptors=[interceptor]) self._server = server @@ -247,21 +262,12 @@ class OpenTelemetryObservabilityTest(unittest.TestCase): self._validate_metrics_exist(self.all_metrics) self._validate_all_metrics_names(self.all_metrics) - def testThrowErrorWhenCallingMultipleInit(self): - otel_plugin = BaseTestOpenTelemetryPlugin(self._provider) - with self.assertRaises(ValueError): - with grpc_observability.OpenTelemetryObservability( - plugins=[otel_plugin] - ) as o11y: - grpc._observability.observability_init(o11y) - def testRecordUnaryUnaryClientOnly(self): server, port = _test_server.start_server() self._server = server - otel_plugin = BaseTestOpenTelemetryPlugin(self._provider) - with grpc_observability.OpenTelemetryObservability( - plugins=[otel_plugin] + with grpc_observability.OpenTelemetryPlugin( + meter_provider=self._provider ): _test_server.unary_unary_call(port=port) @@ -274,9 +280,8 @@ class OpenTelemetryObservabilityTest(unittest.TestCase): self.assertEqual(len(self.all_metrics), 0) server.stop(0) - otel_plugin = BaseTestOpenTelemetryPlugin(self._provider) - with grpc_observability.OpenTelemetryObservability( - plugins=[otel_plugin] + with grpc_observability.OpenTelemetryPlugin( + meter_provider=self._provider ): server, port = _test_server.start_server() self._server = server @@ -286,9 +291,8 @@ class OpenTelemetryObservabilityTest(unittest.TestCase): self._validate_all_metrics_names(self.all_metrics) def testNoRecordAfterExitUseContextManager(self): - otel_plugin = BaseTestOpenTelemetryPlugin(self._provider) - with grpc_observability.OpenTelemetryObservability( - plugins=[otel_plugin] + with grpc_observability.OpenTelemetryPlugin( + meter_provider=self._provider ): server, port = _test_server.start_server() self._server = server @@ -304,16 +308,16 @@ class OpenTelemetryObservabilityTest(unittest.TestCase): self._validate_metrics_exist(self.all_metrics) def testNoRecordAfterExitUseGlobal(self): - otel_plugin = BaseTestOpenTelemetryPlugin(self._provider) - - grpc_observability.start_open_telemetry_observability( - plugins=[otel_plugin] + otel_plugin = grpc_observability.OpenTelemetryPlugin( + meter_provider=self._provider ) + otel_plugin.register_global() + server, port = _test_server.start_server() self._server = server self._port = port _test_server.unary_unary_call(port=port) - grpc_observability.end_open_telemetry_observability() + otel_plugin.deregister_global() self._validate_metrics_exist(self.all_metrics) self._validate_all_metrics_names(self.all_metrics) @@ -324,10 +328,8 @@ class OpenTelemetryObservabilityTest(unittest.TestCase): self._validate_metrics_exist(self.all_metrics) def testRecordUnaryStream(self): - otel_plugin = BaseTestOpenTelemetryPlugin(self._provider) - - with grpc_observability.OpenTelemetryObservability( - plugins=[otel_plugin] + with grpc_observability.OpenTelemetryPlugin( + meter_provider=self._provider ): server, port = _test_server.start_server() self._server = server @@ -337,10 +339,8 @@ class OpenTelemetryObservabilityTest(unittest.TestCase): self._validate_all_metrics_names(self.all_metrics) def testRecordStreamUnary(self): - otel_plugin = BaseTestOpenTelemetryPlugin(self._provider) - - with grpc_observability.OpenTelemetryObservability( - plugins=[otel_plugin] + with grpc_observability.OpenTelemetryPlugin( + meter_provider=self._provider ): server, port = _test_server.start_server() self._server = server @@ -350,10 +350,8 @@ class OpenTelemetryObservabilityTest(unittest.TestCase): self._validate_all_metrics_names(self.all_metrics) def testRecordStreamStream(self): - otel_plugin = BaseTestOpenTelemetryPlugin(self._provider) - - with grpc_observability.OpenTelemetryObservability( - plugins=[otel_plugin] + with grpc_observability.OpenTelemetryPlugin( + meter_provider=self._provider ): server, port = _test_server.start_server() self._server = server @@ -374,11 +372,8 @@ class OpenTelemetryObservabilityTest(unittest.TestCase): return False return True - otel_plugin = BaseTestOpenTelemetryPlugin(self._provider) - otel_plugin.target_attribute_filter = target_filter - - with grpc_observability.OpenTelemetryObservability( - plugins=[otel_plugin] + with grpc_observability.OpenTelemetryPlugin( + meter_provider=self._provider, target_attribute_filter=target_filter ): _test_server.unary_unary_call(port=main_port) _test_server.unary_unary_call(port=backup_port) @@ -406,11 +401,9 @@ class OpenTelemetryObservabilityTest(unittest.TestCase): return False return True - otel_plugin = BaseTestOpenTelemetryPlugin(self._provider) - otel_plugin.generic_method_attribute_filter = method_filter - - with grpc_observability.OpenTelemetryObservability( - plugins=[otel_plugin] + with grpc_observability.OpenTelemetryPlugin( + meter_provider=self._provider, + generic_method_attribute_filter=method_filter, ): server, port = _test_server.start_server() self._server = server From db51a3f69ad6de2d4cce0f7642b829e0ebf6909e Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Thu, 14 Mar 2024 10:49:31 -0700 Subject: [PATCH 44/52] Mark grpc generators for editions support. Edition 2023 makes no changes to method/service descriptors, so these should be unaffected and already supported. PiperOrigin-RevId: 615832918 --- include/grpcpp/impl/codegen/config_protobuf.h | 9 +++++++++ src/compiler/cpp_plugin.h | 14 +++++++++++++- src/compiler/csharp_plugin.cc | 15 ++++++++++++++- src/compiler/objective_c_plugin.cc | 15 ++++++++++++++- src/compiler/php_plugin.cc | 15 ++++++++++++++- src/compiler/python_generator.cc | 4 ---- src/compiler/python_generator.h | 17 ++++++++++++++++- src/compiler/ruby_plugin.cc | 15 ++++++++++++++- 8 files changed, 94 insertions(+), 10 deletions(-) diff --git a/include/grpcpp/impl/codegen/config_protobuf.h b/include/grpcpp/impl/codegen/config_protobuf.h index aa189215d4f..06561cf1210 100644 --- a/include/grpcpp/impl/codegen/config_protobuf.h +++ b/include/grpcpp/impl/codegen/config_protobuf.h @@ -40,8 +40,14 @@ #ifndef GRPC_CUSTOM_DESCRIPTOR #include #include +#if GOOGLE_PROTOBUF_VERSION >= 4025000 +#define GRPC_PROTOBUF_EDITION_SUPPORT +#endif #define GRPC_CUSTOM_DESCRIPTOR ::google::protobuf::Descriptor #define GRPC_CUSTOM_DESCRIPTORPOOL ::google::protobuf::DescriptorPool +#ifdef GRPC_PROTOBUF_EDITION_SUPPORT +#define GRPC_CUSTOM_EDITION ::google::protobuf::Edition +#endif #define GRPC_CUSTOM_FIELDDESCRIPTOR ::google::protobuf::FieldDescriptor #define GRPC_CUSTOM_FILEDESCRIPTOR ::google::protobuf::FileDescriptor #define GRPC_CUSTOM_FILEDESCRIPTORPROTO ::google::protobuf::FileDescriptorProto @@ -85,6 +91,9 @@ typedef GRPC_CUSTOM_MESSAGELITE MessageLite; typedef GRPC_CUSTOM_DESCRIPTOR Descriptor; typedef GRPC_CUSTOM_DESCRIPTORPOOL DescriptorPool; typedef GRPC_CUSTOM_DESCRIPTORDATABASE DescriptorDatabase; +#ifdef GRPC_PROTOBUF_EDITION_SUPPORT +typedef GRPC_CUSTOM_EDITION Edition; +#endif typedef GRPC_CUSTOM_FIELDDESCRIPTOR FieldDescriptor; typedef GRPC_CUSTOM_FILEDESCRIPTOR FileDescriptor; typedef GRPC_CUSTOM_FILEDESCRIPTORPROTO FileDescriptorProto; diff --git a/src/compiler/cpp_plugin.h b/src/compiler/cpp_plugin.h index 3d6f19e44cb..9c8c4bd2474 100644 --- a/src/compiler/cpp_plugin.h +++ b/src/compiler/cpp_plugin.h @@ -34,8 +34,20 @@ class CppGrpcGenerator : public grpc::protobuf::compiler::CodeGenerator { virtual ~CppGrpcGenerator() {} uint64_t GetSupportedFeatures() const override { - return FEATURE_PROTO3_OPTIONAL; + return FEATURE_PROTO3_OPTIONAL +#ifdef GRPC_PROTOBUF_EDITION_SUPPORT + | FEATURE_SUPPORTS_EDITIONS +#endif + ; } +#ifdef GRPC_PROTOBUF_EDITION_SUPPORT + grpc::protobuf::Edition GetMinimumEdition() const override { + return grpc::protobuf::Edition::EDITION_PROTO2; + } + grpc::protobuf::Edition GetMaximumEdition() const override { + return grpc::protobuf::Edition::EDITION_2023; + } +#endif virtual bool Generate(const grpc::protobuf::FileDescriptor* file, const std::string& parameter, diff --git a/src/compiler/csharp_plugin.cc b/src/compiler/csharp_plugin.cc index b4fdb39c353..a7a2e74deb9 100644 --- a/src/compiler/csharp_plugin.cc +++ b/src/compiler/csharp_plugin.cc @@ -30,9 +30,22 @@ class CSharpGrpcGenerator : public grpc::protobuf::compiler::CodeGenerator { ~CSharpGrpcGenerator() {} uint64_t GetSupportedFeatures() const override { - return FEATURE_PROTO3_OPTIONAL; + return FEATURE_PROTO3_OPTIONAL +#ifdef GRPC_PROTOBUF_EDITION_SUPPORT + | FEATURE_SUPPORTS_EDITIONS +#endif + ; } +#ifdef GRPC_PROTOBUF_EDITION_SUPPORT + grpc::protobuf::Edition GetMinimumEdition() const override { + return grpc::protobuf::Edition::EDITION_PROTO2; + } + grpc::protobuf::Edition GetMaximumEdition() const override { + return grpc::protobuf::Edition::EDITION_2023; + } +#endif + bool Generate(const grpc::protobuf::FileDescriptor* file, const std::string& parameter, grpc::protobuf::compiler::GeneratorContext* context, diff --git a/src/compiler/objective_c_plugin.cc b/src/compiler/objective_c_plugin.cc index e13693227fa..4b0f2964212 100644 --- a/src/compiler/objective_c_plugin.cc +++ b/src/compiler/objective_c_plugin.cc @@ -81,8 +81,21 @@ class ObjectiveCGrpcGenerator : public grpc::protobuf::compiler::CodeGenerator { public: uint64_t GetSupportedFeatures() const override { - return FEATURE_PROTO3_OPTIONAL; + return FEATURE_PROTO3_OPTIONAL +#ifdef GRPC_PROTOBUF_EDITION_SUPPORT + | FEATURE_SUPPORTS_EDITIONS +#endif + ; + } + +#ifdef GRPC_PROTOBUF_EDITION_SUPPORT + grpc::protobuf::Edition GetMinimumEdition() const override { + return grpc::protobuf::Edition::EDITION_PROTO2; } + grpc::protobuf::Edition GetMaximumEdition() const override { + return grpc::protobuf::Edition::EDITION_2023; + } +#endif virtual bool Generate(const grpc::protobuf::FileDescriptor* file, const ::std::string& parameter, diff --git a/src/compiler/php_plugin.cc b/src/compiler/php_plugin.cc index 7d4e4ce32c5..9fafe52678c 100644 --- a/src/compiler/php_plugin.cc +++ b/src/compiler/php_plugin.cc @@ -34,9 +34,22 @@ class PHPGrpcGenerator : public grpc::protobuf::compiler::CodeGenerator { ~PHPGrpcGenerator() {} uint64_t GetSupportedFeatures() const override { - return FEATURE_PROTO3_OPTIONAL; + return FEATURE_PROTO3_OPTIONAL +#ifdef GRPC_PROTOBUF_EDITION_SUPPORT + | FEATURE_SUPPORTS_EDITIONS +#endif + ; } +#ifdef GRPC_PROTOBUF_EDITION_SUPPORT + grpc::protobuf::Edition GetMinimumEdition() const override { + return grpc::protobuf::Edition::EDITION_PROTO2; + } + grpc::protobuf::Edition GetMaximumEdition() const override { + return grpc::protobuf::Edition::EDITION_2023; + } +#endif + bool Generate(const grpc::protobuf::FileDescriptor* file, const std::string& parameter, grpc::protobuf::compiler::GeneratorContext* context, diff --git a/src/compiler/python_generator.cc b/src/compiler/python_generator.cc index 53b6083c670..e968fedcbad 100644 --- a/src/compiler/python_generator.cc +++ b/src/compiler/python_generator.cc @@ -944,10 +944,6 @@ static bool ParseParameters(const std::string& parameter, return true; } -uint64_t PythonGrpcGenerator::GetSupportedFeatures() const { - return FEATURE_PROTO3_OPTIONAL; -} - bool PythonGrpcGenerator::Generate(const FileDescriptor* file, const std::string& parameter, GeneratorContext* context, diff --git a/src/compiler/python_generator.h b/src/compiler/python_generator.h index d65f310f64d..1ea0e0a0968 100644 --- a/src/compiler/python_generator.h +++ b/src/compiler/python_generator.h @@ -46,7 +46,22 @@ class PythonGrpcGenerator : public grpc::protobuf::compiler::CodeGenerator { PythonGrpcGenerator(const GeneratorConfiguration& config); ~PythonGrpcGenerator(); - uint64_t GetSupportedFeatures() const override; + uint64_t GetSupportedFeatures() const override { + return FEATURE_PROTO3_OPTIONAL +#ifdef GRPC_PROTOBUF_EDITION_SUPPORT + | FEATURE_SUPPORTS_EDITIONS +#endif + ; + } + +#ifdef GRPC_PROTOBUF_EDITION_SUPPORT + grpc::protobuf::Edition GetMinimumEdition() const override { + return grpc::protobuf::Edition::EDITION_PROTO2; + } + grpc::protobuf::Edition GetMaximumEdition() const override { + return grpc::protobuf::Edition::EDITION_2023; + } +#endif bool Generate(const grpc::protobuf::FileDescriptor* file, const std::string& parameter, diff --git a/src/compiler/ruby_plugin.cc b/src/compiler/ruby_plugin.cc index 8821e613bae..c70206c6b37 100644 --- a/src/compiler/ruby_plugin.cc +++ b/src/compiler/ruby_plugin.cc @@ -30,9 +30,22 @@ class RubyGrpcGenerator : public grpc::protobuf::compiler::CodeGenerator { ~RubyGrpcGenerator() {} uint64_t GetSupportedFeatures() const override { - return FEATURE_PROTO3_OPTIONAL; + return FEATURE_PROTO3_OPTIONAL +#ifdef GRPC_PROTOBUF_EDITION_SUPPORT + | FEATURE_SUPPORTS_EDITIONS +#endif + ; } +#ifdef GRPC_PROTOBUF_EDITION_SUPPORT + grpc::protobuf::Edition GetMinimumEdition() const override { + return grpc::protobuf::Edition::EDITION_PROTO2; + } + grpc::protobuf::Edition GetMaximumEdition() const override { + return grpc::protobuf::Edition::EDITION_2023; + } +#endif + bool Generate(const grpc::protobuf::FileDescriptor* file, const std::string& /*parameter*/, grpc::protobuf::compiler::GeneratorContext* context, From d8f0f64ee61ea57bb07a33491c10702ae645b24f Mon Sep 17 00:00:00 2001 From: Vignesh Babu Date: Thu, 14 Mar 2024 15:20:19 -0700 Subject: [PATCH 45/52] [chaotic-good] Remove ref to server side call early (#36090) Closes #36090 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/36090 from Vignesh2208:chaotic-good-fix 984ec1cdc637569b847936f6769cf93ff0122dfe PiperOrigin-RevId: 615920404 --- .../chaotic_good/server_transport.cc | 108 ++++++++++++------ .../transport/chaotic_good/server_transport.h | 8 +- src/core/lib/transport/call_spine.h | 2 + 3 files changed, 77 insertions(+), 41 deletions(-) diff --git a/src/core/ext/transport/chaotic_good/server_transport.cc b/src/core/ext/transport/chaotic_good/server_transport.cc index c6e07ffd151..50e6ee3f9dd 100644 --- a/src/core/ext/transport/chaotic_good/server_transport.cc +++ b/src/core/ext/transport/chaotic_good/server_transport.cc @@ -71,7 +71,8 @@ auto ChaoticGoodServerTransport::TransportWriteLoop( } auto ChaoticGoodServerTransport::PushFragmentIntoCall( - CallInitiator call_initiator, ClientFragmentFrame frame) { + CallInitiator call_initiator, ClientFragmentFrame frame, + uint32_t stream_id) { auto& headers = frame.headers; return TrySeq( If( @@ -88,44 +89,61 @@ auto ChaoticGoodServerTransport::PushFragmentIntoCall( }, []() -> StatusFlag { return Success{}; }); }, - [call_initiator, - end_of_stream = frame.end_of_stream]() mutable -> StatusFlag { - if (end_of_stream) call_initiator.FinishSends(); + [this, call_initiator, end_of_stream = frame.end_of_stream, + stream_id]() mutable -> StatusFlag { + if (end_of_stream) { + call_initiator.FinishSends(); + // We have received end_of_stream. It is now safe to remove the call + // from the stream map. + MutexLock lock(&mu_); + stream_map_.erase(stream_id); + } return Success{}; }); } auto ChaoticGoodServerTransport::MaybePushFragmentIntoCall( absl::optional call_initiator, absl::Status error, - ClientFragmentFrame frame) { + ClientFragmentFrame frame, uint32_t stream_id) { return If( call_initiator.has_value() && error.ok(), - [this, &call_initiator, &frame]() { + [this, &call_initiator, &frame, &stream_id]() { return Map( call_initiator->SpawnWaitable( "push-fragment", - [call_initiator, frame = std::move(frame), this]() mutable { - return call_initiator->CancelIfFails( - PushFragmentIntoCall(*call_initiator, std::move(frame))); + [call_initiator, frame = std::move(frame), stream_id, + this]() mutable { + return call_initiator->CancelIfFails(PushFragmentIntoCall( + *call_initiator, std::move(frame), stream_id)); }), [](StatusFlag status) { return StatusCast(status); }); }, [&error, &frame]() { - gpr_log(GPR_INFO, - "CHAOTIC_GOOD: Cannot pass frame to stream. Error:%s Frame:%s", - error.ToString().c_str(), frame.ToString().c_str()); + // EOF frames may arrive after the call_initiator's OnDone callback + // has been invoked. In that case, the call_initiator would have + // already been removed from the stream_map and hence the EOF frame + // cannot be pushed into the call. No need to log such frames. + if (!frame.end_of_stream) { + gpr_log( + GPR_INFO, + "CHAOTIC_GOOD: Cannot pass frame to stream. Error:%s Frame:%s", + error.ToString().c_str(), frame.ToString().c_str()); + } return Immediate(std::move(error)); }); } auto ChaoticGoodServerTransport::SendFragment( - ServerFragmentFrame frame, MpscSender outgoing_frames) { + ServerFragmentFrame frame, MpscSender outgoing_frames, + CallInitiator call_initiator) { if (grpc_chaotic_good_trace.enabled()) { gpr_log(GPR_INFO, "CHAOTIC_GOOD: SendFragment: frame=%s", frame.ToString().c_str()); } + // Capture the call_initiator to ensure the underlying call spine is alive + // until the outgoing_frames.Send promise completes. return Map(outgoing_frames.Send(std::move(frame)), - [](bool success) -> absl::Status { + [call_initiator](bool success) -> absl::Status { if (!success) { // Failed to send outgoing frame. return absl::UnavailableError("Transport closed."); @@ -139,24 +157,27 @@ auto ChaoticGoodServerTransport::SendCallBody( CallInitiator call_initiator) { // Continuously send client frame with client to server // messages. - return ForEach(OutgoingMessages(call_initiator), - [stream_id, outgoing_frames, aligned_bytes = aligned_bytes_]( - MessageHandle message) mutable { - ServerFragmentFrame frame; - // Construct frame header (flags, header_length - // and trailer_length will be added in - // serialization). - const uint32_t message_length = message->payload()->Length(); - const uint32_t padding = - message_length % aligned_bytes == 0 - ? 0 - : aligned_bytes - message_length % aligned_bytes; - GPR_ASSERT((message_length + padding) % aligned_bytes == 0); - frame.message = FragmentMessage(std::move(message), padding, - message_length); - frame.stream_id = stream_id; - return SendFragment(std::move(frame), outgoing_frames); - }); + return ForEach( + OutgoingMessages(call_initiator), + // Capture the call_initator to ensure the underlying call + // spine is alive until the SendFragment promise completes. + [stream_id, outgoing_frames, call_initiator, + aligned_bytes = aligned_bytes_](MessageHandle message) mutable { + ServerFragmentFrame frame; + // Construct frame header (flags, header_length + // and trailer_length will be added in + // serialization). + const uint32_t message_length = message->payload()->Length(); + const uint32_t padding = + message_length % aligned_bytes == 0 + ? 0 + : aligned_bytes - message_length % aligned_bytes; + GPR_ASSERT((message_length + padding) % aligned_bytes == 0); + frame.message = + FragmentMessage(std::move(message), padding, message_length); + frame.stream_id = stream_id; + return SendFragment(std::move(frame), outgoing_frames, call_initiator); + }); } auto ChaoticGoodServerTransport::SendCallInitialMetadataAndBody( @@ -179,7 +200,8 @@ auto ChaoticGoodServerTransport::SendCallInitialMetadataAndBody( frame.headers = std::move(*md); frame.stream_id = stream_id; return TrySeq( - SendFragment(std::move(frame), outgoing_frames), + SendFragment(std::move(frame), outgoing_frames, + call_initiator), SendCallBody(stream_id, outgoing_frames, call_initiator)); }, []() { return absl::OkStatus(); }); @@ -201,11 +223,15 @@ auto ChaoticGoodServerTransport::CallOutboundLoop( return Empty{}; }), call_initiator.PullServerTrailingMetadata(), - [stream_id, outgoing_frames](ServerMetadataHandle md) mutable { + // Capture the call_initator to ensure the underlying call_spine + // is alive until the SendFragment promise completes. + [stream_id, outgoing_frames, + call_initiator](ServerMetadataHandle md) mutable { ServerFragmentFrame frame; frame.trailers = std::move(md); frame.stream_id = stream_id; - return SendFragment(std::move(frame), outgoing_frames); + return SendFragment(std::move(frame), outgoing_frames, + call_initiator); }); } @@ -247,7 +273,8 @@ auto ChaoticGoodServerTransport::DeserializeAndPushFragmentToNewCall( } } return MaybePushFragmentIntoCall(std::move(call_initiator), std::move(status), - std::move(fragment_frame)); + std::move(fragment_frame), + frame_header.stream_id); } auto ChaoticGoodServerTransport::DeserializeAndPushFragmentToExistingCall( @@ -262,7 +289,8 @@ auto ChaoticGoodServerTransport::DeserializeAndPushFragmentToExistingCall( frame_header, std::move(buffers), arena, fragment_frame, FrameLimits{1024 * 1024 * 1024, aligned_bytes_ - 1}); return MaybePushFragmentIntoCall(std::move(call_initiator), std::move(status), - std::move(fragment_frame)); + std::move(fragment_frame), + frame_header.stream_id); } auto ChaoticGoodServerTransport::ReadOneFrame(ChaoticGoodTransport& transport) { @@ -421,7 +449,11 @@ absl::Status ChaoticGoodServerTransport::NewStream( if (stream_id <= last_seen_new_stream_id_) { return absl::InternalError("Stream id is not increasing"); } - stream_map_.emplace(stream_id, std::move(call_initiator)); + stream_map_.emplace(stream_id, call_initiator); + call_initiator.OnDone([this, stream_id]() { + MutexLock lock(&mu_); + stream_map_.erase(stream_id); + }); return absl::OkStatus(); } diff --git a/src/core/ext/transport/chaotic_good/server_transport.h b/src/core/ext/transport/chaotic_good/server_transport.h index 021975d9f15..385dc6cf0b7 100644 --- a/src/core/ext/transport/chaotic_good/server_transport.h +++ b/src/core/ext/transport/chaotic_good/server_transport.h @@ -115,7 +115,8 @@ class ChaoticGoodServerTransport final : public Transport, auto SendCallBody(uint32_t stream_id, MpscSender outgoing_frames, CallInitiator call_initiator); static auto SendFragment(ServerFragmentFrame frame, - MpscSender outgoing_frames); + MpscSender outgoing_frames, + CallInitiator call_initiator); auto CallOutboundLoop(uint32_t stream_id, CallInitiator call_initiator); auto OnTransportActivityDone(absl::string_view activity); auto TransportReadLoop(RefCountedPtr transport); @@ -133,9 +134,10 @@ class ChaoticGoodServerTransport final : public Transport, FrameHeader frame_header, BufferPair buffers, ChaoticGoodTransport& transport); auto MaybePushFragmentIntoCall(absl::optional call_initiator, - absl::Status error, ClientFragmentFrame frame); + absl::Status error, ClientFragmentFrame frame, + uint32_t stream_id); auto PushFragmentIntoCall(CallInitiator call_initiator, - ClientFragmentFrame frame); + ClientFragmentFrame frame, uint32_t stream_id); Acceptor* acceptor_ = nullptr; InterActivityLatch got_acceptor_; diff --git a/src/core/lib/transport/call_spine.h b/src/core/lib/transport/call_spine.h index 1c38575c8b2..51568b4160c 100644 --- a/src/core/lib/transport/call_spine.h +++ b/src/core/lib/transport/call_spine.h @@ -296,6 +296,8 @@ class CallInitiator { spine_->Cancel(ServerMetadataFromStatus(absl::CancelledError())); } + void OnDone(absl::AnyInvocable fn) { spine_->OnDone(std::move(fn)); } + template void SpawnGuarded(absl::string_view name, PromiseFactory promise_factory) { spine_->SpawnGuarded(name, std::move(promise_factory)); From e06e45f2af390144a164d50879f16d0bc108e4bc Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Thu, 14 Mar 2024 15:31:01 -0700 Subject: [PATCH 46/52] [OTel C++] Do not add otel_plugin_test with CMake if the otel plugin is not enabled (#36128) This should fix our interop builds Closes #36128 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/36128 from yashykt:OTelPluginCIFix 98165a868b0b0c9060b179a1636afdb65976f15d PiperOrigin-RevId: 615923868 --- CMakeLists.txt | 3 +-- build_autogenerated.yaml | 3 ++- templates/CMakeLists.txt.template | 8 ++++++- templates/README.md | 1 + tools/buildgen/build_cleaner.py | 1 + .../extract_metadata_from_bazel_xml.py | 17 ++++++++++--- tools/buildgen/plugins/check_attrs.py | 1 + tools/run_tests/generated/tests.json | 24 ------------------- 8 files changed, 27 insertions(+), 31 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index afaffdda098..3ef168cffd9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1238,7 +1238,6 @@ if(gRPC_BUILD_TESTS) add_dependencies(buildtests_cxx orca_service_end2end_test) add_dependencies(buildtests_cxx orphanable_test) add_dependencies(buildtests_cxx osa_distance_test) - add_dependencies(buildtests_cxx otel_plugin_test) add_dependencies(buildtests_cxx out_of_bounds_bad_client_test) add_dependencies(buildtests_cxx outlier_detection_lb_config_parser_test) add_dependencies(buildtests_cxx outlier_detection_test) @@ -20620,7 +20619,7 @@ target_link_libraries(osa_distance_test endif() -if(gRPC_BUILD_TESTS) +if(gRPC_BUILD_TESTS AND gRPC_BUILD_GRPCPP_OTEL_PLUGIN) add_executable(otel_plugin_test ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/echo.pb.cc diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index 8f3765d9398..adb59a57a16 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -13210,8 +13210,9 @@ targets: - gtest - name: otel_plugin_test gtest: true - build: test + build: plugin_test language: c++ + plugin_option: gRPC_BUILD_GRPCPP_OTEL_PLUGIN headers: - src/cpp/ext/otel/key_value_iterable.h - src/cpp/ext/otel/otel_call_tracer.h diff --git a/templates/CMakeLists.txt.template b/templates/CMakeLists.txt.template index 3e69ad92258..ad184e021a1 100644 --- a/templates/CMakeLists.txt.template +++ b/templates/CMakeLists.txt.template @@ -200,7 +200,7 @@ def is_generate_cmake_target(lib_or_target): """Returns True if a cmake target should be generated for given library/target.""" # TODO(jtattermusch): extract the metadata to a centralized location. - if lib_or_target.build not in ["all", "plugin", "protoc", "tool", "test", "private"]: + if lib_or_target.build not in ["all", "plugin", "plugin_test", "protoc", "tool", "test", "private"]: return False if lib_or_target.boringssl: # Don't generate target for boringssl libs or tests @@ -775,6 +775,12 @@ ${cc_binary(tgt)} endif() + % elif tgt.build in ["plugin_test"]: + if(gRPC_BUILD_TESTS AND ${tgt.plugin_option}) + <%block filter='platforms_condition_block(tgt.platforms)'> + ${cc_binary(tgt)} + + endif() % elif tgt.build in ["protoc"]: if(gRPC_BUILD_CODEGEN AND gRPC_BUILD_${tgt.name.upper()}) <%block filter='platforms_condition_block(tgt.platforms)'> diff --git a/templates/README.md b/templates/README.md index 03e5836d133..87685a11ad0 100644 --- a/templates/README.md +++ b/templates/README.md @@ -91,6 +91,7 @@ Currently, the "`build`" tag have these meanings: * `"all"`: library to build on `"make all"`, and install on the system. * `"plugin"`: library to build on `"make all"`, and install, but the corresponding CMake option defaults to off. The option needs to be declared manually at present to allow depending on third-party dependencies. * `"protoc"`: a protoc plugin to build on `"make all"` and install on the system. +* `"plugin_test"`: A test that should only be built if the associated plugin is enabled. The plugin is mentioned in the `"plugin_option"` tag. * `"private"`: a library to only build for tests. * `"test"`: a test binary to run on `"make test"`. * `"tool"`: a binary to be built upon `"make tools"`. diff --git a/tools/buildgen/build_cleaner.py b/tools/buildgen/build_cleaner.py index 6071ec50535..e2546fab85e 100755 --- a/tools/buildgen/build_cleaner.py +++ b/tools/buildgen/build_cleaner.py @@ -39,6 +39,7 @@ _ELEM_KEYS = [ "build", "run", "language", + "plugin_option", "public_headers", "headers", "src", diff --git a/tools/buildgen/extract_metadata_from_bazel_xml.py b/tools/buildgen/extract_metadata_from_bazel_xml.py index bcf57fab2c4..d261c96efb2 100755 --- a/tools/buildgen/extract_metadata_from_bazel_xml.py +++ b/tools/buildgen/extract_metadata_from_bazel_xml.py @@ -536,7 +536,10 @@ def update_test_metadata_with_transitive_metadata( """Patches test build metadata with transitive metadata.""" for lib_name, lib_dict in list(all_extra_metadata.items()): # Skip if it isn't not an test - if lib_dict.get("build") != "test" or lib_dict.get("_TYPE") != "target": + if ( + lib_dict.get("build") != "test" + and lib_dict.get("build") != "plugin_test" + ) or lib_dict.get("_TYPE") != "target": continue bazel_rule = bazel_rules[_get_bazel_label(lib_name)] @@ -752,6 +755,7 @@ def _convert_to_build_yaml_like(lib_dict: BuildMetadata) -> BuildYaml: lib_name for lib_name in list(lib_dict.keys()) if lib_dict[lib_name].get("_TYPE", "library") == "test" + or lib_dict[lib_name].get("_TYPE", "library") == "plugin_test" ] # list libraries and targets in predefined order @@ -1006,7 +1010,7 @@ def _generate_build_extra_metadata_for_tests( " to %s" % (test_name, long_name) ) test_metadata[test_name]["_RENAME"] = long_name - + print(test_metadata["test/cpp/ext/otel:otel_plugin_test"]) return test_metadata @@ -1301,6 +1305,13 @@ _BUILD_EXTRA_METADATA = { "_TYPE": "target", "_RENAME": "grpc_cli", }, + "test/cpp/ext/otel:otel_plugin_test": { + "language": "c++", + "build": "plugin_test", + "_TYPE": "target", + "plugin_option": "gRPC_BUILD_GRPCPP_OTEL_PLUGIN", + "_RENAME": "otel_plugin_test", + }, # TODO(jtattermusch): create_jwt and verify_jwt breaks distribtests because it depends on grpc_test_utils and thus requires tests to be built # For now it's ok to disable them as these binaries aren't very useful anyway. # 'test/core/security:create_jwt': { 'language': 'c', 'build': 'tool', '_TYPE': 'target', '_RENAME': 'grpc_create_jwt' }, @@ -1418,10 +1429,10 @@ if "@com_google_protobuf//third_party/utf8_range:utf8_range" not in bazel_rules: "@com_google_protobuf//third_party/utf8_range:utf8_range" ] _BUILD_EXTRA_METADATA["@utf8_range//:utf8_range"] = md -all_extra_metadata.update(_BUILD_EXTRA_METADATA) all_extra_metadata.update( _generate_build_extra_metadata_for_tests(tests, bazel_rules) ) +all_extra_metadata.update(_BUILD_EXTRA_METADATA) # Step 4: Compute the build metadata that will be used in the final build.yaml. # The final build metadata includes transitive dependencies, and sources/headers diff --git a/tools/buildgen/plugins/check_attrs.py b/tools/buildgen/plugins/check_attrs.py index 8f14277f433..6d051f36466 100644 --- a/tools/buildgen/plugins/check_attrs.py +++ b/tools/buildgen/plugins/check_attrs.py @@ -85,6 +85,7 @@ VALID_ATTRIBUTE_KEYS_MAP = { "language": one_of(("c", "c89", "c++", "csharp")), "maxlen": anything(), "platforms": subset_of(("linux", "mac", "posix", "windows")), + "plugin_option": anything(), "run": one_of((True, False)), "secure": one_of(("check", True, False)), "src": anything(), diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 8965c31bb25..49f6bb44395 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -6455,30 +6455,6 @@ ], "uses_polling": true }, - { - "args": [], - "benchmark": false, - "ci_platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "gtest": true, - "language": "c++", - "name": "otel_plugin_test", - "platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "uses_polling": true - }, { "args": [], "benchmark": false, From 930b3934bca2424e8bc95ee70519bcbd1da95ddf Mon Sep 17 00:00:00 2001 From: Jean byroot Boussier Date: Fri, 15 Mar 2024 09:24:45 -0700 Subject: [PATCH 47/52] ruby: register grpc_rb_sStatus as a global variable (#36125) Ref: https://bugs.ruby-lang.org/issues/20311 C global variable that contain references to Ruby object MUST be declared to the Ruby GC to prevent these objects from being collected or moved. There are a few exceptions to that, such as classes defined using the C APIs such as `rb_define_class` etc. Up to Ruby 3.4 however, there was a bug that caused classes created from Ruby with the `Struct.new("Name")` API to also be marked as immortal and immovable. GRPC has been relying on this bug, which I fixed in Ruby 3.4, and now GRPC is crashing when `Struct::Status` is moved around by the GC. ``` -- C level backtrace information ------------------------------------------- ruby(rb_print_backtrace+0x14) [0x5577db219d41] ruby-3.4.0/vm_dump.c:820 ruby(rb_vm_bugreport) ruby-3.4.0/vm_dump.c:1151 ruby(rb_bug_for_fatal_signal+0xfc) [0x5577db3cc60c] ruby-3.4.0/error.c:1066 ruby(sigsegv+0x4d) [0x5577db16358d] ruby-3.4.0/signal.c:926 libc.so.6(0x7f5bacd32520) [0x7f5bacd32520] ruby(rb_class_superclass+0x32) [0x5577db0a9152] ruby-3.4.0/object.c:2239 ruby(struct_ivar_get+0x2a) [0x5577db194e02] ruby-3.4.0/struct.c:49 ruby(struct_ivar_get) ruby-3.4.0/struct.c:40 ruby(num_members) ruby-3.4.0/struct.c:705 ruby(rb_struct_new+0x56) [0x5577db19a9d6] ruby-3.4.0/struct.c:848 lib/grpc/grpc_c.so(grpc_run_batch_stack_build_result+0xe6) [0x7f5b84bb6b96] /tmp/bundle/ruby/3.4.0+0/bundler/gems/grpc-5ed33ee673e3/src/ruby/ext/grpc/rb_call.c:780 lib/grpc/grpc_c.so(grpc_rb_call_run_batch_try) /tmp/bundle/ruby/3.4.0+0/bundler/gems/grpc-5ed33ee673e3/src/ruby/ext/grpc/rb_call.c:839 ruby(rb_ensure+0x10c) [0x5577db007d3c] ruby-3.4.0/eval.c:1000 /tmp/bundle/ruby/3.4.0+0/bundler/gems/grpc-5ed33ee673e3/src/ruby/lib/grpc/grpc_c.so(grpc_rb_call_run_batch+0xca) [0x7f5b84bb595a] /tmp/bundle/ruby/3.4.0+0/bundler/gems/grpc-5ed33ee673e3/src/ruby/ext/grpc/rb_call.c:893 ruby(vm_call_cfunc_with_frame_+0x117) [0x5577db1f4c77] ruby-3.4.0/vm_insnhelper.c:3524 ``` cc @apolcyn @peterzhu2118 @k0kubun Closes #36125 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/36125 from Shopify:fix-ruby-3.4-compat 7a127599c8385a1c3794c7788e034c41710a8f9b PiperOrigin-RevId: 616152904 --- src/ruby/ext/grpc/rb_grpc.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ruby/ext/grpc/rb_grpc.c b/src/ruby/ext/grpc/rb_grpc.c index 4896323bc02..01c4777d800 100644 --- a/src/ruby/ext/grpc/rb_grpc.c +++ b/src/ruby/ext/grpc/rb_grpc.c @@ -467,6 +467,7 @@ void Init_grpc_c() { grpc_rb_mGrpcCore = rb_define_module_under(grpc_rb_mGRPC, "Core"); grpc_rb_sNewServerRpc = rb_struct_define( "NewServerRpc", "method", "host", "deadline", "metadata", "call", NULL); + rb_global_variable(&grpc_rb_sStatus); grpc_rb_sStatus = rb_const_get(rb_cStruct, rb_intern("Status")); sym_code = ID2SYM(rb_intern("code")); sym_details = ID2SYM(rb_intern("details")); From 8d35a303f20429724cbaa50cbfa4e4863d7d5ecd Mon Sep 17 00:00:00 2001 From: Arvind Bright Date: Fri, 15 Mar 2024 10:30:31 -0700 Subject: [PATCH 48/52] update grpc go v1.62.x to patch release (#36061) Closes #36061 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/36061 from arvindbr8:go_patch_release_1.61.1 d89073c5549906b9ac543c88ad9546f98860f1e2 PiperOrigin-RevId: 616173467 --- tools/interop_matrix/client_matrix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/interop_matrix/client_matrix.py b/tools/interop_matrix/client_matrix.py index 00d07b2b243..c66369efd03 100644 --- a/tools/interop_matrix/client_matrix.py +++ b/tools/interop_matrix/client_matrix.py @@ -297,7 +297,7 @@ LANG_RELEASE_MATRIX = { ("v1.59.0", ReleaseInfo(runtimes=["go1.19"])), ("v1.60.1", ReleaseInfo(runtimes=["go1.19"])), ("v1.61.0", ReleaseInfo(runtimes=["go1.19"])), - ("v1.62.0", ReleaseInfo(runtimes=["go1.19"])), + ("v1.62.1", ReleaseInfo(runtimes=["go1.19"])), ] ), "java": OrderedDict( From 070327677050efab2899963ecb1ad072ac46fcb2 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Fri, 15 Mar 2024 14:05:46 -0700 Subject: [PATCH 49/52] [CI] Bazel upgrade (#36104) Upgraded Bazel as follows; - Bazel 6.4 to 6.5 - Bazel 7.0 to 7.1 Closes #36104 PiperOrigin-RevId: 616237591 --- .bazelversion | 2 +- bazel/supported_versions.txt | 4 ++-- bazel/update_mirror.sh | 12 ++++++------ doc/bazel_support.md | 4 ++-- templates/tools/dockerfile/oss_fuzz_base.include | 4 ++-- .../bazelify_tests/dockerimage_current_versions.bzl | 8 ++++---- .../bazelify_tests/test/supported_bazel_versions.bzl | 4 ++-- tools/dockerfile/test/bazel.current_version | 2 +- tools/dockerfile/test/bazel/Dockerfile | 6 +++--- tools/dockerfile/test/bazel_arm64.current_version | 2 +- tools/dockerfile/test/bazel_arm64/Dockerfile | 2 +- .../test/binder_transport_apk.current_version | 2 +- .../dockerfile/test/binder_transport_apk/Dockerfile | 6 +++--- tools/dockerfile/test/sanity.current_version | 2 +- tools/dockerfile/test/sanity/Dockerfile | 2 +- 15 files changed, 31 insertions(+), 31 deletions(-) diff --git a/.bazelversion b/.bazelversion index 19b860c1872..f22d756da39 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -6.4.0 +6.5.0 diff --git a/bazel/supported_versions.txt b/bazel/supported_versions.txt index 1197e4958b6..73d8dc3a64c 100644 --- a/bazel/supported_versions.txt +++ b/bazel/supported_versions.txt @@ -1,2 +1,2 @@ -6.4.0 -7.0.0 +6.5.0 +7.1.0 diff --git a/bazel/update_mirror.sh b/bazel/update_mirror.sh index c5f6c84b487..98d752a3cff 100755 --- a/bazel/update_mirror.sh +++ b/bazel/update_mirror.sh @@ -57,12 +57,12 @@ function upload { # upload "github.com/google/boringssl/archive/1c2769383f027befac5b75b6cedd25daf3bf4dcf.tar.gz" # bazel binaries used by the tools/bazel wrapper script -upload github.com/bazelbuild/bazel/releases/download/6.4.0/bazel-6.4.0-linux-x86_64 -upload github.com/bazelbuild/bazel/releases/download/6.4.0/bazel-6.4.0-darwin-x86_64 -upload github.com/bazelbuild/bazel/releases/download/6.4.0/bazel-6.4.0-windows-x86_64.exe -upload github.com/bazelbuild/bazel/releases/download/7.0.0/bazel-7.0.0-linux-x86_64 -upload github.com/bazelbuild/bazel/releases/download/7.0.0/bazel-7.0.0-darwin-x86_64 -upload github.com/bazelbuild/bazel/releases/download/7.0.0/bazel-7.0.0-windows-x86_64.exe +upload github.com/bazelbuild/bazel/releases/download/6.5.0/bazel-6.5.0-linux-x86_64 +upload github.com/bazelbuild/bazel/releases/download/6.5.0/bazel-6.5.0-darwin-x86_64 +upload github.com/bazelbuild/bazel/releases/download/6.5.0/bazel-6.5.0-windows-x86_64.exe +upload github.com/bazelbuild/bazel/releases/download/7.1.0/bazel-7.1.0-linux-x86_64 +upload github.com/bazelbuild/bazel/releases/download/7.1.0/bazel-7.1.0-darwin-x86_64 +upload github.com/bazelbuild/bazel/releases/download/7.1.0/bazel-7.1.0-windows-x86_64.exe # Collect the github archives to mirror from grpc_deps.bzl grep -o '"https://github.com/[^"]*"' bazel/grpc_deps.bzl | sed 's/^"https:\/\///' | sed 's/"$//' | while read -r line ; do diff --git a/doc/bazel_support.md b/doc/bazel_support.md index 0fbc3c22a5f..08cb014f106 100644 --- a/doc/bazel_support.md +++ b/doc/bazel_support.md @@ -43,7 +43,7 @@ However individual releases may have a broader compatibility range. The currently supported versions are captured by the following list: -- [`6.4.0`](https://github.com/bazelbuild/bazel/releases/tag/6.4.0) -- [`7.0.0`](https://github.com/bazelbuild/bazel/releases/tag/7.0.0) +- [`6.5.0`](https://github.com/bazelbuild/bazel/releases/tag/6.5.0) +- [`7.1.0`](https://github.com/bazelbuild/bazel/releases/tag/7.1.0) NOTE: gRPC doesn't support bzlmod yet. \ No newline at end of file diff --git a/templates/tools/dockerfile/oss_fuzz_base.include b/templates/tools/dockerfile/oss_fuzz_base.include index 126b58b723e..95bd410dfb2 100644 --- a/templates/tools/dockerfile/oss_fuzz_base.include +++ b/templates/tools/dockerfile/oss_fuzz_base.include @@ -1,8 +1,8 @@ # Pinned version of the base image is used to avoid regressions caused # by rebuilding of this docker image. To see available versions, you can run # "gcloud container images list-tags gcr.io/oss-fuzz-base/base-builder" -# Image(fd89316ac4c5) is built on Aug 17, 2023 -FROM gcr.io/oss-fuzz-base/base-builder@sha256:fd89316ac4c5f3e25802ca95a00062cece14f0602c5512d71ffeedc22734c0b9 +# This base image is built on Mar 12, 2024 +FROM gcr.io/oss-fuzz-base/base-builder@sha256:c3581153788bc49f3634fec3cd36a5d6dfd26632c4afc157fb6faf8ce3af732e # -------------------------- WARNING -------------------------------------- # If you are making changes to this file, consider changing diff --git a/tools/bazelify_tests/dockerimage_current_versions.bzl b/tools/bazelify_tests/dockerimage_current_versions.bzl index 70dcfc40ae5..41b9a5707b5 100644 --- a/tools/bazelify_tests/dockerimage_current_versions.bzl +++ b/tools/bazelify_tests/dockerimage_current_versions.bzl @@ -88,9 +88,9 @@ DOCKERIMAGE_CURRENT_VERSIONS = { "tools/dockerfile/interoptest/grpc_interop_ruby.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_ruby@sha256:efd7f41a736dd4b8f73b32f5215b86f6bfe9013c422dfcd77978de0253aaee45", "tools/dockerfile/interoptest/lb_interop_fake_servers.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/lb_interop_fake_servers@sha256:b89a51dd9147e1293f50ee64dd719fce5929ca7894d3770a3d80dbdecb99fd52", "tools/dockerfile/test/android_ndk.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/android_ndk@sha256:ab154ecb062af2111d2d3550c4d3da3384201d9893bbd37d49e8160fc34bc137", - "tools/dockerfile/test/bazel.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/bazel@sha256:32bde2dcb2087f2a32afab59e4dfedf7e8c76a52c69881f63a239d311f0e5ecf", - "tools/dockerfile/test/bazel_arm64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/bazel_arm64@sha256:3b087387c44dee405c1b80d6ff50994e6d8e90a4ef67cc94b4291f1a29c0ef41", - "tools/dockerfile/test/binder_transport_apk.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/binder_transport_apk@sha256:bf60a187cd2ce1abe8b4f32ae6479040a72ca6aa789cd5ab509f60ceb37a41f9", + "tools/dockerfile/test/bazel.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/bazel@sha256:eb327f8e44f2712f557de1d8918c41c3cba1112c4b39b13104c29211ed8f827a", + "tools/dockerfile/test/bazel_arm64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/bazel_arm64@sha256:6cddc0ecdb42a7db7105b73fc3192edb911702102d1bac671e26d44a17d7aa95", + "tools/dockerfile/test/binder_transport_apk.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/binder_transport_apk@sha256:a680a8b7d645a2c25948ad3f82f6380c8a1e13fcfe74fc3569acb3b0b202851e", "tools/dockerfile/test/csharp_debian11_arm64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/csharp_debian11_arm64@sha256:4d4bc5f15e03f3d3d8fd889670ecde2c66a2e4d2dd9db80733c05c1d90c8a248", "tools/dockerfile/test/csharp_debian11_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/csharp_debian11_x64@sha256:b2e5c47d986312ea0850e2f2e696b45d23ee0aabceea161d31e28559e19ec4a5", "tools/dockerfile/test/cxx_alpine_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/cxx_alpine_x64@sha256:f2019edf9f2afd5042567f11afb1aa78a789fc9acdcce5ee0c14cc11f6830ed7", @@ -112,5 +112,5 @@ DOCKERIMAGE_CURRENT_VERSIONS = { "tools/dockerfile/test/rbe_ubuntu2004.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/rbe_ubuntu2004@sha256:d3951aeadf43e3bee6adc5b86d26cdaf0b9d1b5baf790d7b2530d1c197adc9f8", "tools/dockerfile/test/ruby_debian11_arm64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_arm64@sha256:d2e79919b2e2d4cc36a29682ecb5170641df4fb506cfb453978ffdeb8a841bd9", "tools/dockerfile/test/ruby_debian11_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_x64@sha256:f8fc0ec22065278e5bc02ad7f9a68191e46d083035b3a90ed587561dba9c58c5", - "tools/dockerfile/test/sanity.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/sanity@sha256:ca65b29302e8e381aabf9deb1952281f4e9963654e19c3d681148b5aa972c8a2", + "tools/dockerfile/test/sanity.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/sanity@sha256:a83cf1344c166834d76f49e08448f4253b535cdbc47efffb195cfa710f5114d6", } diff --git a/tools/bazelify_tests/test/supported_bazel_versions.bzl b/tools/bazelify_tests/test/supported_bazel_versions.bzl index d83471d3061..e53bf51d06b 100644 --- a/tools/bazelify_tests/test/supported_bazel_versions.bzl +++ b/tools/bazelify_tests/test/supported_bazel_versions.bzl @@ -17,6 +17,6 @@ This file is generated from the supported_bazel_versions.bzl.template """ SUPPORTED_BAZEL_VERSIONS = [ - "6.4.0", - "7.0.0", + "6.5.0", + "7.1.0", ] diff --git a/tools/dockerfile/test/bazel.current_version b/tools/dockerfile/test/bazel.current_version index 972682e1522..39fe083900c 100644 --- a/tools/dockerfile/test/bazel.current_version +++ b/tools/dockerfile/test/bazel.current_version @@ -1 +1 @@ -us-docker.pkg.dev/grpc-testing/testing-images-public/bazel:cacad91746cd598d8756de89b912be291de1f019@sha256:32bde2dcb2087f2a32afab59e4dfedf7e8c76a52c69881f63a239d311f0e5ecf \ No newline at end of file +us-docker.pkg.dev/grpc-testing/testing-images-public/bazel:71afcbd2698751336bd9890d00eb37ed790b3ac6@sha256:eb327f8e44f2712f557de1d8918c41c3cba1112c4b39b13104c29211ed8f827a \ No newline at end of file diff --git a/tools/dockerfile/test/bazel/Dockerfile b/tools/dockerfile/test/bazel/Dockerfile index c52f8313cbb..b85b8a207d6 100644 --- a/tools/dockerfile/test/bazel/Dockerfile +++ b/tools/dockerfile/test/bazel/Dockerfile @@ -15,8 +15,8 @@ # Pinned version of the base image is used to avoid regressions caused # by rebuilding of this docker image. To see available versions, you can run # "gcloud container images list-tags gcr.io/oss-fuzz-base/base-builder" -# Image(fd89316ac4c5) is built on Aug 17, 2023 -FROM gcr.io/oss-fuzz-base/base-builder@sha256:fd89316ac4c5f3e25802ca95a00062cece14f0602c5512d71ffeedc22734c0b9 +# This base image is built on Mar 12, 2024 +FROM gcr.io/oss-fuzz-base/base-builder@sha256:c3581153788bc49f3634fec3cd36a5d6dfd26632c4afc157fb6faf8ce3af732e # -------------------------- WARNING -------------------------------------- # If you are making changes to this file, consider changing @@ -38,7 +38,7 @@ RUN apt-get update && apt-get -y install \ # Bazel installation # Must be in sync with tools/bazel -ENV BAZEL_VERSION 6.4.0 +ENV BAZEL_VERSION 6.5.0 # The correct bazel version is already preinstalled, no need to use //tools/bazel wrapper. ENV DISABLE_BAZEL_WRAPPER 1 diff --git a/tools/dockerfile/test/bazel_arm64.current_version b/tools/dockerfile/test/bazel_arm64.current_version index 71008d44e60..eccc25d1444 100644 --- a/tools/dockerfile/test/bazel_arm64.current_version +++ b/tools/dockerfile/test/bazel_arm64.current_version @@ -1 +1 @@ -us-docker.pkg.dev/grpc-testing/testing-images-public/bazel_arm64:2638335cbe0cbd8f77c54affdf0e4b36f7741785@sha256:3b087387c44dee405c1b80d6ff50994e6d8e90a4ef67cc94b4291f1a29c0ef41 \ No newline at end of file +us-docker.pkg.dev/grpc-testing/testing-images-public/bazel_arm64:415b428a3108de92d1d819cd053181a110a8078b@sha256:6cddc0ecdb42a7db7105b73fc3192edb911702102d1bac671e26d44a17d7aa95 \ No newline at end of file diff --git a/tools/dockerfile/test/bazel_arm64/Dockerfile b/tools/dockerfile/test/bazel_arm64/Dockerfile index afc341b6ad9..8efb0ef1b22 100644 --- a/tools/dockerfile/test/bazel_arm64/Dockerfile +++ b/tools/dockerfile/test/bazel_arm64/Dockerfile @@ -97,7 +97,7 @@ RUN apt-get update && apt-get -y install libc++-dev clang && apt-get clean # Bazel installation # Must be in sync with tools/bazel -ENV BAZEL_VERSION 6.4.0 +ENV BAZEL_VERSION 6.5.0 # The correct bazel version is already preinstalled, no need to use //tools/bazel wrapper. ENV DISABLE_BAZEL_WRAPPER 1 diff --git a/tools/dockerfile/test/binder_transport_apk.current_version b/tools/dockerfile/test/binder_transport_apk.current_version index 439b81ec9dd..c5fbd60b28b 100644 --- a/tools/dockerfile/test/binder_transport_apk.current_version +++ b/tools/dockerfile/test/binder_transport_apk.current_version @@ -1 +1 @@ -us-docker.pkg.dev/grpc-testing/testing-images-public/binder_transport_apk:783671f597502a2dc3ebf190d52d55be70836a33@sha256:bf60a187cd2ce1abe8b4f32ae6479040a72ca6aa789cd5ab509f60ceb37a41f9 \ No newline at end of file +us-docker.pkg.dev/grpc-testing/testing-images-public/binder_transport_apk:fbe8a32dd2006fab456f2fa0d0e82d3979b7f30b@sha256:a680a8b7d645a2c25948ad3f82f6380c8a1e13fcfe74fc3569acb3b0b202851e \ No newline at end of file diff --git a/tools/dockerfile/test/binder_transport_apk/Dockerfile b/tools/dockerfile/test/binder_transport_apk/Dockerfile index c0644d3285b..9f38130a87a 100644 --- a/tools/dockerfile/test/binder_transport_apk/Dockerfile +++ b/tools/dockerfile/test/binder_transport_apk/Dockerfile @@ -15,8 +15,8 @@ # Pinned version of the base image is used to avoid regressions caused # by rebuilding of this docker image. To see available versions, you can run # "gcloud container images list-tags gcr.io/oss-fuzz-base/base-builder" -# Image(fd89316ac4c5) is built on Aug 17, 2023 -FROM gcr.io/oss-fuzz-base/base-builder@sha256:fd89316ac4c5f3e25802ca95a00062cece14f0602c5512d71ffeedc22734c0b9 +# This base image is built on Mar 12, 2024 +FROM gcr.io/oss-fuzz-base/base-builder@sha256:c3581153788bc49f3634fec3cd36a5d6dfd26632c4afc157fb6faf8ce3af732e # -------------------------- WARNING -------------------------------------- # If you are making changes to this file, consider changing @@ -38,7 +38,7 @@ RUN apt-get update && apt-get -y install \ # Bazel installation # Must be in sync with tools/bazel -ENV BAZEL_VERSION 6.4.0 +ENV BAZEL_VERSION 6.5.0 # The correct bazel version is already preinstalled, no need to use //tools/bazel wrapper. ENV DISABLE_BAZEL_WRAPPER 1 diff --git a/tools/dockerfile/test/sanity.current_version b/tools/dockerfile/test/sanity.current_version index 2a2d514f070..f05e8d51456 100644 --- a/tools/dockerfile/test/sanity.current_version +++ b/tools/dockerfile/test/sanity.current_version @@ -1 +1 @@ -us-docker.pkg.dev/grpc-testing/testing-images-public/sanity:2d20232b40465f3487deef87274f4bfd7122b201@sha256:ca65b29302e8e381aabf9deb1952281f4e9963654e19c3d681148b5aa972c8a2 \ No newline at end of file +us-docker.pkg.dev/grpc-testing/testing-images-public/sanity:38e0bdb9e26335d2d947263d33efa0f688dcd2fc@sha256:a83cf1344c166834d76f49e08448f4253b535cdbc47efffb195cfa710f5114d6 \ No newline at end of file diff --git a/tools/dockerfile/test/sanity/Dockerfile b/tools/dockerfile/test/sanity/Dockerfile index b44b5582710..c75124fa18a 100644 --- a/tools/dockerfile/test/sanity/Dockerfile +++ b/tools/dockerfile/test/sanity/Dockerfile @@ -108,7 +108,7 @@ RUN apt-get update && apt-get install -y jq git && apt-get clean # Bazel installation # Must be in sync with tools/bazel -ENV BAZEL_VERSION 6.4.0 +ENV BAZEL_VERSION 6.5.0 # The correct bazel version is already preinstalled, no need to use //tools/bazel wrapper. ENV DISABLE_BAZEL_WRAPPER 1 From 2aa1b6479f900ae9eaafbf0876567920308808cf Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Fri, 15 Mar 2024 14:35:24 -0700 Subject: [PATCH 50/52] [Build] Massage grpcio_tools build configuration for protobuf upgrade. (#36124) This is a prerequisite for the upcoming Protobuf v26 upgrade and has the following changes; - Reduced the optimization level from `O3` to `O1` on Linux to workaround a segfault issue on `manylinux2014/x86`. This is believed to be caused by an aggressive but buggy optimization that `gcc` 10 made so workaround is to reduce the optimization level. `gcsio_tools` isn't that performance sensitive so we should be able to afford this change. (internal b/329134877) - Added a Windows library dependency `Shell32.lib` to support the use of `CommandLineToArgvW`. (internal b/328455319) Closes #36124 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/36124 from veblush:grpcio-x e7b555bfcf031a6aa126de8a89148056fc5724b0 PiperOrigin-RevId: 616245600 --- tools/distrib/python/grpcio_tools/setup.py | 25 ++++++++++++---------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py index b778bc31ada..a41915895b3 100644 --- a/tools/distrib/python/grpcio_tools/setup.py +++ b/tools/distrib/python/grpcio_tools/setup.py @@ -23,7 +23,6 @@ import subprocess from subprocess import PIPE import sys import sysconfig -import traceback import setuptools from setuptools import Extension @@ -152,14 +151,7 @@ class BuildExt(build_ext.build_ext): self.compiler._compile = new_compile - try: - build_ext.build_ext.build_extensions(self) - except Exception as error: - formatted_exception = traceback.format_exc() - support.diagnose_build_ext_error(self, error, formatted_exception) - raise CommandError( - "Failed `build_ext` step:\n{}".format(formatted_exception) - ) + build_ext.build_ext.build_extensions(self) # There are some situations (like on Windows) where CC, CFLAGS, and LDFLAGS are @@ -179,12 +171,23 @@ if EXTRA_ENV_COMPILE_ARGS is None: # We need to statically link the C++ Runtime, only the C runtime is # available dynamically EXTRA_ENV_COMPILE_ARGS += " /MT" - elif "linux" in sys.platform or "darwin" in sys.platform: - # GCC & Clang by defaults uses C17 so only C++14 needs to be specified. + elif "linux" in sys.platform: + # GCC by defaults uses C17 so only C++14 needs to be specified. + EXTRA_ENV_COMPILE_ARGS += " -std=c++14" + EXTRA_ENV_COMPILE_ARGS += " -fno-wrapv -frtti" + # Reduce the optimization level from O3 (in many cases) to O1 to + # workaround gcc misalignment bug with MOVAPS (internal b/329134877) + EXTRA_ENV_COMPILE_ARGS += " -O1" + elif "darwin" in sys.platform: + # AppleClang by defaults uses C17 so only C++14 needs to be specified. EXTRA_ENV_COMPILE_ARGS += " -std=c++14" EXTRA_ENV_COMPILE_ARGS += " -fno-wrapv -frtti" + EXTRA_ENV_COMPILE_ARGS += " -stdlib=libc++ -DHAVE_UNISTD_H" if EXTRA_ENV_LINK_ARGS is None: EXTRA_ENV_LINK_ARGS = "" + # This is needed for protobuf/main.cc + if "win32" in sys.platform: + EXTRA_ENV_LINK_ARGS += " Shell32.lib" # NOTE(rbellevi): Clang on Mac OS will make all static symbols (both # variables and objects) global weak symbols. When a process loads the # protobuf wheel's shared object library before loading *this* C extension, From d6edb33ae7baf56b67c4fd974278cbde4550e35c Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Fri, 15 Mar 2024 16:00:10 -0700 Subject: [PATCH 51/52] [Clean-up] Removed grpc.gyp (#36077) No more gyp file! Closes #36077 PiperOrigin-RevId: 616267538 --- .gitattributes | 2 - grpc.gyp | 2702 ----------------------------------- templates/grpc.gyp.template | 162 --- 3 files changed, 2866 deletions(-) delete mode 100644 grpc.gyp delete mode 100644 templates/grpc.gyp.template diff --git a/.gitattributes b/.gitattributes index 3d72aae39fd..b03b218eb51 100644 --- a/.gitattributes +++ b/.gitattributes @@ -12,10 +12,8 @@ gRPC-ProtoRPC.podspec linguist-generated=true gRPC-RxLibrary.podspec linguist-generated=true gRPC.podspec linguist-generated=true grpc.gemspec linguist-generated=true -grpc.gyp linguist-generated=true grpc.def linguist-generated=true package.xml linguist-generated=true -binding.gyp linguist-generated=true Package.swift linguist-generated=true src/python/grpcio/grpc_core_dependencies.py linguist-generated=true src/ruby/ext/grpc/rb_grpc_imports.generated.h linguist-generated=true diff --git a/grpc.gyp b/grpc.gyp deleted file mode 100644 index eb70fc54080..00000000000 --- a/grpc.gyp +++ /dev/null @@ -1,2702 +0,0 @@ -# GRPC GYP build file - -# This file has been automatically generated from a template file. -# Please look at the templates directory instead. -# This file can be regenerated from the template by running -# tools/buildgen/generate_projects.sh - -# Copyright 2015 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{ - 'variables': { - # The openssl and zlib dependencies must be passed in as variables - # defined in an included gypi file, usually common.gypi. - 'openssl_gyp_target%': 'Please Define openssl_gyp_target variable', - 'zlib_gyp_target%': 'Please Define zlib_gyp_target variable', - - 'grpc_gcov%': 'false', - 'grpc_alpine%': 'false', - }, - 'target_defaults': { - 'configurations': { - 'Debug': { - 'cflags': [ - '-O0', - ], - 'defines': [ - '_DEBUG', - 'DEBUG', - ], - }, - 'Release': { - 'cflags': [ - '-O2', - '-Wframe-larger-than=16384', - ], - 'defines': [ - 'NDEBUG', - ], - }, - }, - 'cflags': [ - '-g', - '-Wall', - '-Wextra', - '-DOSATOMIC_USE_INLINED=1', - '-Ithird_party/abseil-cpp', - '-Ithird_party/re2', - '-Ithird_party/upb', - '-Isrc/core/ext/upb-gen', - '-Isrc/core/ext/upbdefs-gen', - '-Ithird_party/utf8_range', - '-Ithird_party/xxhash', - '-Ithird_party/cares/cares/include', - '-Ithird_party/cares', - '-Ithird_party/cares/cares', - '-Ithird_party/address_sorting/include', - ], - 'ldflags': [ - '-g', - ], - 'cflags_c': [ - '-Werror', - '-std=c11', - ], - 'cflags_cc': [ - '-Werror', - '-std=c++14', - ], - 'include_dirs': [ - '.', - '../..', - 'include', - ], - 'defines': [ - 'GRPC_ARES=0', - ], - 'dependencies': [ - '<(openssl_gyp_target)', - '<(zlib_gyp_target)', - ], - 'conditions': [ - ['grpc_gcov=="true"', { - 'cflags': [ - '-O0', - '-fprofile-arcs', - '-ftest-coverage', - '-Wno-return-type', - ], - 'defines': [ - '_DEBUG', - 'DEBUG', - 'GPR_GCOV', - ], - 'ldflags': [ - '-fprofile-arcs', - '-ftest-coverage', - '-rdynamic', - '-lstdc++', - ], - }], - ['grpc_alpine=="true"', { - 'defines': [ - 'GPR_MUSL_LIBC_COMPAT' - ] - }], - ['OS == "win"', { - 'defines': [ - '_WIN32_WINNT=0x0600', - 'WIN32_LEAN_AND_MEAN', - '_HAS_EXCEPTIONS=0', - 'UNICODE', - '_UNICODE', - 'NOMINMAX', - ], - 'msvs_settings': { - 'VCCLCompilerTool': { - 'RuntimeLibrary': 1, # static debug - } - }, - "libraries": [ - "ws2_32" - ] - }], - ['OS == "mac"', { - 'xcode_settings': { - 'OTHER_CFLAGS': [ - '-g', - '-Wall', - '-Wextra', - '-DOSATOMIC_USE_INLINED=1', - '-Ithird_party/abseil-cpp', - '-Ithird_party/re2', - '-Ithird_party/upb', - '-Isrc/core/ext/upb-gen', - '-Isrc/core/ext/upbdefs-gen', - '-Ithird_party/utf8_range', - '-Ithird_party/xxhash', - '-Ithird_party/cares/cares/include', - '-Ithird_party/cares', - '-Ithird_party/cares/cares', - '-Ithird_party/address_sorting/include', - ], - 'OTHER_CPLUSPLUSFLAGS': [ - '-g', - '-Wall', - '-Wextra', - '-DOSATOMIC_USE_INLINED=1', - '-Ithird_party/abseil-cpp', - '-Ithird_party/re2', - '-Ithird_party/upb', - '-Isrc/core/ext/upb-gen', - '-Isrc/core/ext/upbdefs-gen', - '-Ithird_party/utf8_range', - '-Ithird_party/xxhash', - '-Ithird_party/cares/cares/include', - '-Ithird_party/cares', - '-Ithird_party/cares/cares', - '-Ithird_party/address_sorting/include', - '-stdlib=libc++', - '-std=c++14', - '-Wno-error=deprecated-declarations', - ], - }, - }] - ] - }, - 'targets': [ - { - 'target_name': 'address_sorting', - 'type': 'static_library', - 'dependencies': [ - ], - 'sources': [ - 'third_party/address_sorting/address_sorting.c', - 'third_party/address_sorting/address_sorting_posix.c', - 'third_party/address_sorting/address_sorting_windows.c', - ], - }, - { - 'target_name': 'gpr', - 'type': 'static_library', - 'dependencies': [ - 'absl/base:base', - 'absl/base:core_headers', - 'absl/base:log_severity', - 'absl/flags:flag', - 'absl/flags:marshalling', - 'absl/functional:any_invocable', - 'absl/log:log', - 'absl/memory:memory', - 'absl/random:random', - 'absl/status:status', - 'absl/strings:cord', - 'absl/strings:str_format', - 'absl/strings:strings', - 'absl/synchronization:synchronization', - 'absl/time:time', - 'absl/types:optional', - 'absl/types:variant', - ], - 'sources': [ - 'src/core/lib/config/config_vars.cc', - 'src/core/lib/config/config_vars_non_generated.cc', - 'src/core/lib/config/load_config.cc', - 'src/core/lib/event_engine/thread_local.cc', - 'src/core/lib/gpr/alloc.cc', - 'src/core/lib/gpr/android/log.cc', - 'src/core/lib/gpr/atm.cc', - 'src/core/lib/gpr/iphone/cpu.cc', - 'src/core/lib/gpr/linux/cpu.cc', - 'src/core/lib/gpr/linux/log.cc', - 'src/core/lib/gpr/log.cc', - 'src/core/lib/gpr/msys/tmpfile.cc', - 'src/core/lib/gpr/posix/cpu.cc', - 'src/core/lib/gpr/posix/log.cc', - 'src/core/lib/gpr/posix/string.cc', - 'src/core/lib/gpr/posix/sync.cc', - 'src/core/lib/gpr/posix/time.cc', - 'src/core/lib/gpr/posix/tmpfile.cc', - 'src/core/lib/gpr/string.cc', - 'src/core/lib/gpr/sync.cc', - 'src/core/lib/gpr/sync_abseil.cc', - 'src/core/lib/gpr/time.cc', - 'src/core/lib/gpr/time_precise.cc', - 'src/core/lib/gpr/windows/cpu.cc', - 'src/core/lib/gpr/windows/log.cc', - 'src/core/lib/gpr/windows/string.cc', - 'src/core/lib/gpr/windows/string_util.cc', - 'src/core/lib/gpr/windows/sync.cc', - 'src/core/lib/gpr/windows/time.cc', - 'src/core/lib/gpr/windows/tmpfile.cc', - 'src/core/lib/gprpp/crash.cc', - 'src/core/lib/gprpp/examine_stack.cc', - 'src/core/lib/gprpp/fork.cc', - 'src/core/lib/gprpp/host_port.cc', - 'src/core/lib/gprpp/linux/env.cc', - 'src/core/lib/gprpp/mpscq.cc', - 'src/core/lib/gprpp/posix/env.cc', - 'src/core/lib/gprpp/posix/stat.cc', - 'src/core/lib/gprpp/posix/thd.cc', - 'src/core/lib/gprpp/strerror.cc', - 'src/core/lib/gprpp/tchar.cc', - 'src/core/lib/gprpp/time_util.cc', - 'src/core/lib/gprpp/windows/env.cc', - 'src/core/lib/gprpp/windows/stat.cc', - 'src/core/lib/gprpp/windows/thd.cc', - ], - }, - { - 'target_name': 'grpc', - 'type': 'static_library', - 'dependencies': [ - 'upb_json_lib', - 'upb_textformat_lib', - 're2', - 'z', - 'absl/algorithm:container', - 'absl/base:config', - 'absl/base:no_destructor', - 'absl/cleanup:cleanup', - 'absl/container:flat_hash_map', - 'absl/container:flat_hash_set', - 'absl/container:inlined_vector', - 'absl/functional:bind_front', - 'absl/functional:function_ref', - 'absl/hash:hash', - 'absl/meta:type_traits', - 'absl/random:bit_gen_ref', - 'absl/random:distributions', - 'absl/status:statusor', - 'absl/types:span', - 'absl/utility:utility', - 'cares', - 'gpr', - 'address_sorting', - ], - 'sources': [ - 'src/core/client_channel/backup_poller.cc', - 'src/core/client_channel/client_channel_channelz.cc', - 'src/core/client_channel/client_channel_factory.cc', - 'src/core/client_channel/client_channel_filter.cc', - 'src/core/client_channel/client_channel_plugin.cc', - 'src/core/client_channel/client_channel_service_config.cc', - 'src/core/client_channel/config_selector.cc', - 'src/core/client_channel/dynamic_filters.cc', - 'src/core/client_channel/global_subchannel_pool.cc', - 'src/core/client_channel/http_proxy_mapper.cc', - 'src/core/client_channel/local_subchannel_pool.cc', - 'src/core/client_channel/retry_filter.cc', - 'src/core/client_channel/retry_filter_legacy_call_data.cc', - 'src/core/client_channel/retry_service_config.cc', - 'src/core/client_channel/retry_throttle.cc', - 'src/core/client_channel/subchannel.cc', - 'src/core/client_channel/subchannel_pool_interface.cc', - 'src/core/client_channel/subchannel_stream_client.cc', - 'src/core/ext/filters/backend_metrics/backend_metric_filter.cc', - 'src/core/ext/filters/census/grpc_context.cc', - 'src/core/ext/filters/channel_idle/channel_idle_filter.cc', - 'src/core/ext/filters/channel_idle/idle_filter_state.cc', - 'src/core/ext/filters/channel_idle/legacy_channel_idle_filter.cc', - 'src/core/ext/filters/deadline/deadline_filter.cc', - 'src/core/ext/filters/fault_injection/fault_injection_filter.cc', - 'src/core/ext/filters/fault_injection/fault_injection_service_config_parser.cc', - 'src/core/ext/filters/http/client/http_client_filter.cc', - 'src/core/ext/filters/http/client_authority_filter.cc', - 'src/core/ext/filters/http/http_filters_plugin.cc', - 'src/core/ext/filters/http/message_compress/compression_filter.cc', - 'src/core/ext/filters/http/message_compress/legacy_compression_filter.cc', - 'src/core/ext/filters/http/server/http_server_filter.cc', - 'src/core/ext/filters/message_size/message_size_filter.cc', - 'src/core/ext/filters/rbac/rbac_filter.cc', - 'src/core/ext/filters/rbac/rbac_service_config_parser.cc', - 'src/core/ext/filters/server_config_selector/server_config_selector_filter.cc', - 'src/core/ext/filters/stateful_session/stateful_session_filter.cc', - 'src/core/ext/filters/stateful_session/stateful_session_service_config_parser.cc', - 'src/core/ext/gcp/metadata_query.cc', - 'src/core/ext/transport/chttp2/alpn/alpn.cc', - 'src/core/ext/transport/chttp2/client/chttp2_connector.cc', - 'src/core/ext/transport/chttp2/server/chttp2_server.cc', - 'src/core/ext/transport/chttp2/transport/bin_decoder.cc', - 'src/core/ext/transport/chttp2/transport/bin_encoder.cc', - 'src/core/ext/transport/chttp2/transport/chttp2_transport.cc', - 'src/core/ext/transport/chttp2/transport/decode_huff.cc', - 'src/core/ext/transport/chttp2/transport/flow_control.cc', - 'src/core/ext/transport/chttp2/transport/frame.cc', - 'src/core/ext/transport/chttp2/transport/frame_data.cc', - 'src/core/ext/transport/chttp2/transport/frame_goaway.cc', - 'src/core/ext/transport/chttp2/transport/frame_ping.cc', - 'src/core/ext/transport/chttp2/transport/frame_rst_stream.cc', - 'src/core/ext/transport/chttp2/transport/frame_settings.cc', - 'src/core/ext/transport/chttp2/transport/frame_window_update.cc', - 'src/core/ext/transport/chttp2/transport/hpack_encoder.cc', - 'src/core/ext/transport/chttp2/transport/hpack_encoder_table.cc', - 'src/core/ext/transport/chttp2/transport/hpack_parse_result.cc', - 'src/core/ext/transport/chttp2/transport/hpack_parser.cc', - 'src/core/ext/transport/chttp2/transport/hpack_parser_table.cc', - 'src/core/ext/transport/chttp2/transport/http2_settings.cc', - 'src/core/ext/transport/chttp2/transport/http_trace.cc', - 'src/core/ext/transport/chttp2/transport/huffsyms.cc', - 'src/core/ext/transport/chttp2/transport/max_concurrent_streams_policy.cc', - 'src/core/ext/transport/chttp2/transport/parsing.cc', - 'src/core/ext/transport/chttp2/transport/ping_abuse_policy.cc', - 'src/core/ext/transport/chttp2/transport/ping_callbacks.cc', - 'src/core/ext/transport/chttp2/transport/ping_rate_policy.cc', - 'src/core/ext/transport/chttp2/transport/stream_lists.cc', - 'src/core/ext/transport/chttp2/transport/varint.cc', - 'src/core/ext/transport/chttp2/transport/write_size_policy.cc', - 'src/core/ext/transport/chttp2/transport/writing.cc', - 'src/core/ext/transport/inproc/inproc_plugin.cc', - 'src/core/ext/transport/inproc/inproc_transport.cc', - 'src/core/ext/transport/inproc/legacy_inproc_transport.cc', - 'src/core/ext/upb-gen/envoy/admin/v3/certs.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/admin/v3/clusters.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/admin/v3/config_dump.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/admin/v3/config_dump_shared.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/admin/v3/init_dump.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/admin/v3/listeners.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/admin/v3/memory.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/admin/v3/metrics.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/admin/v3/mutex_stats.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/admin/v3/server_info.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/admin/v3/tap.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/annotations/deprecation.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/annotations/resource.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/accesslog/v3/accesslog.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/bootstrap/v3/bootstrap.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/cluster/v3/circuit_breaker.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/cluster/v3/cluster.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/cluster/v3/filter.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/cluster/v3/outlier_detection.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/common/matcher/v3/matcher.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/core/v3/address.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/core/v3/backoff.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/core/v3/base.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/core/v3/config_source.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/core/v3/event_service_config.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/core/v3/extension.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/core/v3/grpc_method_list.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/core/v3/grpc_service.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/core/v3/health_check.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/core/v3/http_service.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/core/v3/http_uri.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/core/v3/protocol.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/core/v3/proxy_protocol.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/core/v3/resolver.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/core/v3/socket_option.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/core/v3/substitution_format_string.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/core/v3/udp_socket_config.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/endpoint/v3/endpoint.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/endpoint/v3/endpoint_components.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/endpoint/v3/load_report.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/listener/v3/api_listener.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/listener/v3/listener.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/listener/v3/listener_components.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/listener/v3/quic_config.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/listener/v3/udp_listener_config.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/metrics/v3/metrics_service.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/metrics/v3/stats.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/overload/v3/overload.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/rbac/v3/rbac.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/route/v3/route.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/route/v3/route_components.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/route/v3/scoped_route.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/tap/v3/common.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/trace/v3/datadog.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/trace/v3/dynamic_ot.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/trace/v3/http_tracer.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/trace/v3/lightstep.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/trace/v3/opencensus.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/trace/v3/opentelemetry.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/trace/v3/service.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/trace/v3/skywalking.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/trace/v3/trace.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/trace/v3/xray.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/config/trace/v3/zipkin.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/data/accesslog/v3/accesslog.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/extensions/clusters/aggregate/v3/cluster.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/extensions/filters/common/fault/v3/fault.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/extensions/filters/http/fault/v3/fault.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/extensions/filters/http/rbac/v3/rbac.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/extensions/filters/http/router/v3/router.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/extensions/filters/http/stateful_session/v3/stateful_session.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/extensions/http/stateful_session/cookie/v3/cookie.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/client_side_weighted_round_robin.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/common/v3/common.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/pick_first/v3/pick_first.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/ring_hash/v3/ring_hash.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/wrr_locality/v3/wrr_locality.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/cert.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/common.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/secret.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/tls.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/extensions/upstreams/http/v3/http_protocol_options.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/service/discovery/v3/ads.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/service/discovery/v3/discovery.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/service/load_stats/v3/lrs.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/service/status/v3/csds.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/type/http/v3/cookie.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/type/http/v3/path_transformation.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/type/matcher/v3/filter_state.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/type/matcher/v3/http_inputs.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/type/matcher/v3/metadata.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/type/matcher/v3/node.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/type/matcher/v3/number.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/type/matcher/v3/path.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/type/matcher/v3/regex.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/type/matcher/v3/status_code_input.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/type/matcher/v3/string.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/type/matcher/v3/struct.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/type/matcher/v3/value.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/type/metadata/v3/metadata.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/type/tracing/v3/custom_tag.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/type/v3/hash_policy.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/type/v3/http.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/type/v3/http_status.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/type/v3/percent.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/type/v3/range.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/type/v3/ratelimit_strategy.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/type/v3/ratelimit_unit.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/type/v3/semantic_version.upb_minitable.c', - 'src/core/ext/upb-gen/envoy/type/v3/token_bucket.upb_minitable.c', - 'src/core/ext/upb-gen/google/api/annotations.upb_minitable.c', - 'src/core/ext/upb-gen/google/api/expr/v1alpha1/checked.upb_minitable.c', - 'src/core/ext/upb-gen/google/api/expr/v1alpha1/syntax.upb_minitable.c', - 'src/core/ext/upb-gen/google/api/http.upb_minitable.c', - 'src/core/ext/upb-gen/google/api/httpbody.upb_minitable.c', - 'src/core/ext/upb-gen/google/protobuf/any.upb_minitable.c', - 'src/core/ext/upb-gen/google/protobuf/descriptor.upb_minitable.c', - 'src/core/ext/upb-gen/google/protobuf/duration.upb_minitable.c', - 'src/core/ext/upb-gen/google/protobuf/empty.upb_minitable.c', - 'src/core/ext/upb-gen/google/protobuf/struct.upb_minitable.c', - 'src/core/ext/upb-gen/google/protobuf/timestamp.upb_minitable.c', - 'src/core/ext/upb-gen/google/protobuf/wrappers.upb_minitable.c', - 'src/core/ext/upb-gen/google/rpc/status.upb_minitable.c', - 'src/core/ext/upb-gen/opencensus/proto/trace/v1/trace_config.upb_minitable.c', - 'src/core/ext/upb-gen/src/proto/grpc/gcp/altscontext.upb_minitable.c', - 'src/core/ext/upb-gen/src/proto/grpc/gcp/handshaker.upb_minitable.c', - 'src/core/ext/upb-gen/src/proto/grpc/gcp/transport_security_common.upb_minitable.c', - 'src/core/ext/upb-gen/src/proto/grpc/health/v1/health.upb_minitable.c', - 'src/core/ext/upb-gen/src/proto/grpc/lb/v1/load_balancer.upb_minitable.c', - 'src/core/ext/upb-gen/src/proto/grpc/lookup/v1/rls.upb_minitable.c', - 'src/core/ext/upb-gen/src/proto/grpc/lookup/v1/rls_config.upb_minitable.c', - 'src/core/ext/upb-gen/udpa/annotations/migrate.upb_minitable.c', - 'src/core/ext/upb-gen/udpa/annotations/security.upb_minitable.c', - 'src/core/ext/upb-gen/udpa/annotations/sensitive.upb_minitable.c', - 'src/core/ext/upb-gen/udpa/annotations/status.upb_minitable.c', - 'src/core/ext/upb-gen/udpa/annotations/versioning.upb_minitable.c', - 'src/core/ext/upb-gen/validate/validate.upb_minitable.c', - 'src/core/ext/upb-gen/xds/annotations/v3/migrate.upb_minitable.c', - 'src/core/ext/upb-gen/xds/annotations/v3/security.upb_minitable.c', - 'src/core/ext/upb-gen/xds/annotations/v3/sensitive.upb_minitable.c', - 'src/core/ext/upb-gen/xds/annotations/v3/status.upb_minitable.c', - 'src/core/ext/upb-gen/xds/annotations/v3/versioning.upb_minitable.c', - 'src/core/ext/upb-gen/xds/core/v3/authority.upb_minitable.c', - 'src/core/ext/upb-gen/xds/core/v3/cidr.upb_minitable.c', - 'src/core/ext/upb-gen/xds/core/v3/collection_entry.upb_minitable.c', - 'src/core/ext/upb-gen/xds/core/v3/context_params.upb_minitable.c', - 'src/core/ext/upb-gen/xds/core/v3/extension.upb_minitable.c', - 'src/core/ext/upb-gen/xds/core/v3/resource.upb_minitable.c', - 'src/core/ext/upb-gen/xds/core/v3/resource_locator.upb_minitable.c', - 'src/core/ext/upb-gen/xds/core/v3/resource_name.upb_minitable.c', - 'src/core/ext/upb-gen/xds/data/orca/v3/orca_load_report.upb_minitable.c', - 'src/core/ext/upb-gen/xds/service/orca/v3/orca.upb_minitable.c', - 'src/core/ext/upb-gen/xds/type/matcher/v3/cel.upb_minitable.c', - 'src/core/ext/upb-gen/xds/type/matcher/v3/domain.upb_minitable.c', - 'src/core/ext/upb-gen/xds/type/matcher/v3/http_inputs.upb_minitable.c', - 'src/core/ext/upb-gen/xds/type/matcher/v3/ip.upb_minitable.c', - 'src/core/ext/upb-gen/xds/type/matcher/v3/matcher.upb_minitable.c', - 'src/core/ext/upb-gen/xds/type/matcher/v3/range.upb_minitable.c', - 'src/core/ext/upb-gen/xds/type/matcher/v3/regex.upb_minitable.c', - 'src/core/ext/upb-gen/xds/type/matcher/v3/string.upb_minitable.c', - 'src/core/ext/upb-gen/xds/type/v3/cel.upb_minitable.c', - 'src/core/ext/upb-gen/xds/type/v3/range.upb_minitable.c', - 'src/core/ext/upb-gen/xds/type/v3/typed_struct.upb_minitable.c', - 'src/core/ext/upbdefs-gen/envoy/admin/v3/certs.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/admin/v3/clusters.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/admin/v3/config_dump.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/admin/v3/config_dump_shared.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/admin/v3/init_dump.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/admin/v3/listeners.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/admin/v3/memory.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/admin/v3/metrics.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/admin/v3/mutex_stats.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/admin/v3/server_info.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/admin/v3/tap.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/annotations/deprecation.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/annotations/resource.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/accesslog/v3/accesslog.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/bootstrap/v3/bootstrap.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/cluster/v3/circuit_breaker.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/cluster/v3/cluster.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/cluster/v3/filter.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/cluster/v3/outlier_detection.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/common/matcher/v3/matcher.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/core/v3/address.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/core/v3/backoff.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/core/v3/base.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/core/v3/config_source.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/core/v3/event_service_config.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/core/v3/extension.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/core/v3/grpc_method_list.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/core/v3/grpc_service.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/core/v3/health_check.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/core/v3/http_service.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/core/v3/http_uri.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/core/v3/protocol.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/core/v3/proxy_protocol.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/core/v3/resolver.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/core/v3/socket_option.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/core/v3/substitution_format_string.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/core/v3/udp_socket_config.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/endpoint/v3/endpoint.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/endpoint/v3/endpoint_components.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/endpoint/v3/load_report.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/listener/v3/api_listener.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/listener/v3/listener.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/listener/v3/listener_components.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/listener/v3/quic_config.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/listener/v3/udp_listener_config.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/metrics/v3/metrics_service.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/metrics/v3/stats.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/overload/v3/overload.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/rbac/v3/rbac.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/route/v3/route.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/route/v3/route_components.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/route/v3/scoped_route.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/tap/v3/common.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/trace/v3/datadog.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/trace/v3/dynamic_ot.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/trace/v3/http_tracer.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/trace/v3/lightstep.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/trace/v3/opencensus.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/trace/v3/opentelemetry.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/trace/v3/service.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/trace/v3/skywalking.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/trace/v3/trace.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/trace/v3/xray.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/config/trace/v3/zipkin.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/data/accesslog/v3/accesslog.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/extensions/clusters/aggregate/v3/cluster.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/extensions/filters/common/fault/v3/fault.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/extensions/filters/http/fault/v3/fault.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/extensions/filters/http/rbac/v3/rbac.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/extensions/filters/http/router/v3/router.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/extensions/filters/http/stateful_session/v3/stateful_session.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/extensions/http/stateful_session/cookie/v3/cookie.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/extensions/transport_sockets/tls/v3/cert.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/extensions/transport_sockets/tls/v3/common.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/extensions/transport_sockets/tls/v3/secret.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/extensions/transport_sockets/tls/v3/tls.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/extensions/upstreams/http/v3/http_protocol_options.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/service/discovery/v3/ads.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/service/discovery/v3/discovery.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/service/load_stats/v3/lrs.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/service/status/v3/csds.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/type/http/v3/cookie.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/type/http/v3/path_transformation.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/type/matcher/v3/filter_state.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/type/matcher/v3/http_inputs.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/type/matcher/v3/metadata.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/type/matcher/v3/node.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/type/matcher/v3/number.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/type/matcher/v3/path.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/type/matcher/v3/regex.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/type/matcher/v3/status_code_input.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/type/matcher/v3/string.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/type/matcher/v3/struct.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/type/matcher/v3/value.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/type/metadata/v3/metadata.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/type/tracing/v3/custom_tag.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/type/v3/hash_policy.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/type/v3/http.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/type/v3/http_status.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/type/v3/percent.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/type/v3/range.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/type/v3/ratelimit_strategy.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/type/v3/ratelimit_unit.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/type/v3/semantic_version.upbdefs.c', - 'src/core/ext/upbdefs-gen/envoy/type/v3/token_bucket.upbdefs.c', - 'src/core/ext/upbdefs-gen/google/api/annotations.upbdefs.c', - 'src/core/ext/upbdefs-gen/google/api/expr/v1alpha1/checked.upbdefs.c', - 'src/core/ext/upbdefs-gen/google/api/expr/v1alpha1/syntax.upbdefs.c', - 'src/core/ext/upbdefs-gen/google/api/http.upbdefs.c', - 'src/core/ext/upbdefs-gen/google/api/httpbody.upbdefs.c', - 'src/core/ext/upbdefs-gen/google/protobuf/any.upbdefs.c', - 'src/core/ext/upbdefs-gen/google/protobuf/descriptor.upbdefs.c', - 'src/core/ext/upbdefs-gen/google/protobuf/duration.upbdefs.c', - 'src/core/ext/upbdefs-gen/google/protobuf/empty.upbdefs.c', - 'src/core/ext/upbdefs-gen/google/protobuf/struct.upbdefs.c', - 'src/core/ext/upbdefs-gen/google/protobuf/timestamp.upbdefs.c', - 'src/core/ext/upbdefs-gen/google/protobuf/wrappers.upbdefs.c', - 'src/core/ext/upbdefs-gen/google/rpc/status.upbdefs.c', - 'src/core/ext/upbdefs-gen/opencensus/proto/trace/v1/trace_config.upbdefs.c', - 'src/core/ext/upbdefs-gen/src/proto/grpc/lookup/v1/rls_config.upbdefs.c', - 'src/core/ext/upbdefs-gen/udpa/annotations/migrate.upbdefs.c', - 'src/core/ext/upbdefs-gen/udpa/annotations/security.upbdefs.c', - 'src/core/ext/upbdefs-gen/udpa/annotations/sensitive.upbdefs.c', - 'src/core/ext/upbdefs-gen/udpa/annotations/status.upbdefs.c', - 'src/core/ext/upbdefs-gen/udpa/annotations/versioning.upbdefs.c', - 'src/core/ext/upbdefs-gen/validate/validate.upbdefs.c', - 'src/core/ext/upbdefs-gen/xds/annotations/v3/migrate.upbdefs.c', - 'src/core/ext/upbdefs-gen/xds/annotations/v3/security.upbdefs.c', - 'src/core/ext/upbdefs-gen/xds/annotations/v3/sensitive.upbdefs.c', - 'src/core/ext/upbdefs-gen/xds/annotations/v3/status.upbdefs.c', - 'src/core/ext/upbdefs-gen/xds/annotations/v3/versioning.upbdefs.c', - 'src/core/ext/upbdefs-gen/xds/core/v3/authority.upbdefs.c', - 'src/core/ext/upbdefs-gen/xds/core/v3/cidr.upbdefs.c', - 'src/core/ext/upbdefs-gen/xds/core/v3/collection_entry.upbdefs.c', - 'src/core/ext/upbdefs-gen/xds/core/v3/context_params.upbdefs.c', - 'src/core/ext/upbdefs-gen/xds/core/v3/extension.upbdefs.c', - 'src/core/ext/upbdefs-gen/xds/core/v3/resource.upbdefs.c', - 'src/core/ext/upbdefs-gen/xds/core/v3/resource_locator.upbdefs.c', - 'src/core/ext/upbdefs-gen/xds/core/v3/resource_name.upbdefs.c', - 'src/core/ext/upbdefs-gen/xds/type/matcher/v3/cel.upbdefs.c', - 'src/core/ext/upbdefs-gen/xds/type/matcher/v3/domain.upbdefs.c', - 'src/core/ext/upbdefs-gen/xds/type/matcher/v3/http_inputs.upbdefs.c', - 'src/core/ext/upbdefs-gen/xds/type/matcher/v3/ip.upbdefs.c', - 'src/core/ext/upbdefs-gen/xds/type/matcher/v3/matcher.upbdefs.c', - 'src/core/ext/upbdefs-gen/xds/type/matcher/v3/range.upbdefs.c', - 'src/core/ext/upbdefs-gen/xds/type/matcher/v3/regex.upbdefs.c', - 'src/core/ext/upbdefs-gen/xds/type/matcher/v3/string.upbdefs.c', - 'src/core/ext/upbdefs-gen/xds/type/v3/cel.upbdefs.c', - 'src/core/ext/upbdefs-gen/xds/type/v3/range.upbdefs.c', - 'src/core/ext/upbdefs-gen/xds/type/v3/typed_struct.upbdefs.c', - 'src/core/ext/xds/certificate_provider_store.cc', - 'src/core/ext/xds/file_watcher_certificate_provider_factory.cc', - 'src/core/ext/xds/xds_api.cc', - 'src/core/ext/xds/xds_audit_logger_registry.cc', - 'src/core/ext/xds/xds_bootstrap.cc', - 'src/core/ext/xds/xds_bootstrap_grpc.cc', - 'src/core/ext/xds/xds_certificate_provider.cc', - 'src/core/ext/xds/xds_channel_stack_modifier.cc', - 'src/core/ext/xds/xds_client.cc', - 'src/core/ext/xds/xds_client_grpc.cc', - 'src/core/ext/xds/xds_client_stats.cc', - 'src/core/ext/xds/xds_cluster.cc', - 'src/core/ext/xds/xds_cluster_specifier_plugin.cc', - 'src/core/ext/xds/xds_common_types.cc', - 'src/core/ext/xds/xds_endpoint.cc', - 'src/core/ext/xds/xds_health_status.cc', - 'src/core/ext/xds/xds_http_fault_filter.cc', - 'src/core/ext/xds/xds_http_filters.cc', - 'src/core/ext/xds/xds_http_rbac_filter.cc', - 'src/core/ext/xds/xds_http_stateful_session_filter.cc', - 'src/core/ext/xds/xds_lb_policy_registry.cc', - 'src/core/ext/xds/xds_listener.cc', - 'src/core/ext/xds/xds_route_config.cc', - 'src/core/ext/xds/xds_routing.cc', - 'src/core/ext/xds/xds_server_config_fetcher.cc', - 'src/core/ext/xds/xds_transport_grpc.cc', - 'src/core/lib/address_utils/parse_address.cc', - 'src/core/lib/address_utils/sockaddr_utils.cc', - 'src/core/lib/backoff/backoff.cc', - 'src/core/lib/backoff/random_early_detection.cc', - 'src/core/lib/channel/call_tracer.cc', - 'src/core/lib/channel/channel_args.cc', - 'src/core/lib/channel/channel_args_preconditioning.cc', - 'src/core/lib/channel/channel_stack.cc', - 'src/core/lib/channel/channel_stack_builder.cc', - 'src/core/lib/channel/channel_stack_builder_impl.cc', - 'src/core/lib/channel/channel_stack_trace.cc', - 'src/core/lib/channel/channel_trace.cc', - 'src/core/lib/channel/channelz.cc', - 'src/core/lib/channel/channelz_registry.cc', - 'src/core/lib/channel/connected_channel.cc', - 'src/core/lib/channel/metrics.cc', - 'src/core/lib/channel/promise_based_filter.cc', - 'src/core/lib/channel/server_call_tracer_filter.cc', - 'src/core/lib/channel/status_util.cc', - 'src/core/lib/compression/compression.cc', - 'src/core/lib/compression/compression_internal.cc', - 'src/core/lib/compression/message_compress.cc', - 'src/core/lib/config/core_configuration.cc', - 'src/core/lib/debug/event_log.cc', - 'src/core/lib/debug/histogram_view.cc', - 'src/core/lib/debug/stats.cc', - 'src/core/lib/debug/stats_data.cc', - 'src/core/lib/debug/trace.cc', - 'src/core/lib/event_engine/ares_resolver.cc', - 'src/core/lib/event_engine/cf_engine/cf_engine.cc', - 'src/core/lib/event_engine/cf_engine/cfstream_endpoint.cc', - 'src/core/lib/event_engine/cf_engine/dns_service_resolver.cc', - 'src/core/lib/event_engine/channel_args_endpoint_config.cc', - 'src/core/lib/event_engine/default_event_engine.cc', - 'src/core/lib/event_engine/default_event_engine_factory.cc', - 'src/core/lib/event_engine/event_engine.cc', - 'src/core/lib/event_engine/forkable.cc', - 'src/core/lib/event_engine/posix_engine/ev_epoll1_linux.cc', - 'src/core/lib/event_engine/posix_engine/ev_poll_posix.cc', - 'src/core/lib/event_engine/posix_engine/event_poller_posix_default.cc', - 'src/core/lib/event_engine/posix_engine/internal_errqueue.cc', - 'src/core/lib/event_engine/posix_engine/lockfree_event.cc', - 'src/core/lib/event_engine/posix_engine/native_posix_dns_resolver.cc', - 'src/core/lib/event_engine/posix_engine/posix_endpoint.cc', - 'src/core/lib/event_engine/posix_engine/posix_engine.cc', - 'src/core/lib/event_engine/posix_engine/posix_engine_listener.cc', - 'src/core/lib/event_engine/posix_engine/posix_engine_listener_utils.cc', - 'src/core/lib/event_engine/posix_engine/tcp_socket_utils.cc', - 'src/core/lib/event_engine/posix_engine/timer.cc', - 'src/core/lib/event_engine/posix_engine/timer_heap.cc', - 'src/core/lib/event_engine/posix_engine/timer_manager.cc', - 'src/core/lib/event_engine/posix_engine/traced_buffer_list.cc', - 'src/core/lib/event_engine/posix_engine/wakeup_fd_eventfd.cc', - 'src/core/lib/event_engine/posix_engine/wakeup_fd_pipe.cc', - 'src/core/lib/event_engine/posix_engine/wakeup_fd_posix_default.cc', - 'src/core/lib/event_engine/resolved_address.cc', - 'src/core/lib/event_engine/shim.cc', - 'src/core/lib/event_engine/slice.cc', - 'src/core/lib/event_engine/slice_buffer.cc', - 'src/core/lib/event_engine/tcp_socket_utils.cc', - 'src/core/lib/event_engine/thread_pool/thread_count.cc', - 'src/core/lib/event_engine/thread_pool/thread_pool_factory.cc', - 'src/core/lib/event_engine/thread_pool/work_stealing_thread_pool.cc', - 'src/core/lib/event_engine/thready_event_engine/thready_event_engine.cc', - 'src/core/lib/event_engine/time_util.cc', - 'src/core/lib/event_engine/trace.cc', - 'src/core/lib/event_engine/utils.cc', - 'src/core/lib/event_engine/windows/grpc_polled_fd_windows.cc', - 'src/core/lib/event_engine/windows/iocp.cc', - 'src/core/lib/event_engine/windows/native_windows_dns_resolver.cc', - 'src/core/lib/event_engine/windows/win_socket.cc', - 'src/core/lib/event_engine/windows/windows_endpoint.cc', - 'src/core/lib/event_engine/windows/windows_engine.cc', - 'src/core/lib/event_engine/windows/windows_listener.cc', - 'src/core/lib/event_engine/work_queue/basic_work_queue.cc', - 'src/core/lib/experiments/config.cc', - 'src/core/lib/experiments/experiments.cc', - 'src/core/lib/gprpp/load_file.cc', - 'src/core/lib/gprpp/per_cpu.cc', - 'src/core/lib/gprpp/posix/directory_reader.cc', - 'src/core/lib/gprpp/ref_counted_string.cc', - 'src/core/lib/gprpp/status_helper.cc', - 'src/core/lib/gprpp/time.cc', - 'src/core/lib/gprpp/time_averaged_stats.cc', - 'src/core/lib/gprpp/uuid_v4.cc', - 'src/core/lib/gprpp/validation_errors.cc', - 'src/core/lib/gprpp/windows/directory_reader.cc', - 'src/core/lib/gprpp/work_serializer.cc', - 'src/core/lib/handshaker/proxy_mapper_registry.cc', - 'src/core/lib/http/format_request.cc', - 'src/core/lib/http/httpcli.cc', - 'src/core/lib/http/httpcli_security_connector.cc', - 'src/core/lib/http/parser.cc', - 'src/core/lib/iomgr/buffer_list.cc', - 'src/core/lib/iomgr/call_combiner.cc', - 'src/core/lib/iomgr/cfstream_handle.cc', - 'src/core/lib/iomgr/closure.cc', - 'src/core/lib/iomgr/combiner.cc', - 'src/core/lib/iomgr/dualstack_socket_posix.cc', - 'src/core/lib/iomgr/endpoint.cc', - 'src/core/lib/iomgr/endpoint_cfstream.cc', - 'src/core/lib/iomgr/endpoint_pair_posix.cc', - 'src/core/lib/iomgr/endpoint_pair_windows.cc', - 'src/core/lib/iomgr/error.cc', - 'src/core/lib/iomgr/error_cfstream.cc', - 'src/core/lib/iomgr/ev_apple.cc', - 'src/core/lib/iomgr/ev_epoll1_linux.cc', - 'src/core/lib/iomgr/ev_poll_posix.cc', - 'src/core/lib/iomgr/ev_posix.cc', - 'src/core/lib/iomgr/ev_windows.cc', - 'src/core/lib/iomgr/event_engine_shims/closure.cc', - 'src/core/lib/iomgr/event_engine_shims/endpoint.cc', - 'src/core/lib/iomgr/event_engine_shims/tcp_client.cc', - 'src/core/lib/iomgr/exec_ctx.cc', - 'src/core/lib/iomgr/executor.cc', - 'src/core/lib/iomgr/fork_posix.cc', - 'src/core/lib/iomgr/fork_windows.cc', - 'src/core/lib/iomgr/gethostname_fallback.cc', - 'src/core/lib/iomgr/gethostname_host_name_max.cc', - 'src/core/lib/iomgr/gethostname_sysconf.cc', - 'src/core/lib/iomgr/grpc_if_nametoindex_posix.cc', - 'src/core/lib/iomgr/grpc_if_nametoindex_unsupported.cc', - 'src/core/lib/iomgr/internal_errqueue.cc', - 'src/core/lib/iomgr/iocp_windows.cc', - 'src/core/lib/iomgr/iomgr.cc', - 'src/core/lib/iomgr/iomgr_internal.cc', - 'src/core/lib/iomgr/iomgr_posix.cc', - 'src/core/lib/iomgr/iomgr_posix_cfstream.cc', - 'src/core/lib/iomgr/iomgr_windows.cc', - 'src/core/lib/iomgr/lockfree_event.cc', - 'src/core/lib/iomgr/polling_entity.cc', - 'src/core/lib/iomgr/pollset.cc', - 'src/core/lib/iomgr/pollset_set.cc', - 'src/core/lib/iomgr/pollset_set_windows.cc', - 'src/core/lib/iomgr/pollset_windows.cc', - 'src/core/lib/iomgr/resolve_address.cc', - 'src/core/lib/iomgr/resolve_address_posix.cc', - 'src/core/lib/iomgr/resolve_address_windows.cc', - 'src/core/lib/iomgr/sockaddr_utils_posix.cc', - 'src/core/lib/iomgr/socket_factory_posix.cc', - 'src/core/lib/iomgr/socket_mutator.cc', - 'src/core/lib/iomgr/socket_utils_common_posix.cc', - 'src/core/lib/iomgr/socket_utils_linux.cc', - 'src/core/lib/iomgr/socket_utils_posix.cc', - 'src/core/lib/iomgr/socket_utils_windows.cc', - 'src/core/lib/iomgr/socket_windows.cc', - 'src/core/lib/iomgr/systemd_utils.cc', - 'src/core/lib/iomgr/tcp_client.cc', - 'src/core/lib/iomgr/tcp_client_cfstream.cc', - 'src/core/lib/iomgr/tcp_client_posix.cc', - 'src/core/lib/iomgr/tcp_client_windows.cc', - 'src/core/lib/iomgr/tcp_posix.cc', - 'src/core/lib/iomgr/tcp_server.cc', - 'src/core/lib/iomgr/tcp_server_posix.cc', - 'src/core/lib/iomgr/tcp_server_utils_posix_common.cc', - 'src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.cc', - 'src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.cc', - 'src/core/lib/iomgr/tcp_server_windows.cc', - 'src/core/lib/iomgr/tcp_windows.cc', - 'src/core/lib/iomgr/timer.cc', - 'src/core/lib/iomgr/timer_generic.cc', - 'src/core/lib/iomgr/timer_heap.cc', - 'src/core/lib/iomgr/timer_manager.cc', - 'src/core/lib/iomgr/unix_sockets_posix.cc', - 'src/core/lib/iomgr/unix_sockets_posix_noop.cc', - 'src/core/lib/iomgr/vsock.cc', - 'src/core/lib/iomgr/wakeup_fd_eventfd.cc', - 'src/core/lib/iomgr/wakeup_fd_nospecial.cc', - 'src/core/lib/iomgr/wakeup_fd_pipe.cc', - 'src/core/lib/iomgr/wakeup_fd_posix.cc', - 'src/core/lib/json/json_object_loader.cc', - 'src/core/lib/json/json_reader.cc', - 'src/core/lib/json/json_util.cc', - 'src/core/lib/json/json_writer.cc', - 'src/core/lib/matchers/matchers.cc', - 'src/core/lib/promise/activity.cc', - 'src/core/lib/promise/party.cc', - 'src/core/lib/promise/sleep.cc', - 'src/core/lib/promise/trace.cc', - 'src/core/lib/resource_quota/api.cc', - 'src/core/lib/resource_quota/arena.cc', - 'src/core/lib/resource_quota/connection_quota.cc', - 'src/core/lib/resource_quota/memory_quota.cc', - 'src/core/lib/resource_quota/periodic_update.cc', - 'src/core/lib/resource_quota/resource_quota.cc', - 'src/core/lib/resource_quota/thread_quota.cc', - 'src/core/lib/resource_quota/trace.cc', - 'src/core/lib/security/authorization/audit_logging.cc', - 'src/core/lib/security/authorization/authorization_policy_provider_vtable.cc', - 'src/core/lib/security/authorization/evaluate_args.cc', - 'src/core/lib/security/authorization/grpc_authorization_engine.cc', - 'src/core/lib/security/authorization/grpc_server_authz_filter.cc', - 'src/core/lib/security/authorization/matchers.cc', - 'src/core/lib/security/authorization/rbac_policy.cc', - 'src/core/lib/security/authorization/stdout_logger.cc', - 'src/core/lib/security/certificate_provider/certificate_provider_registry.cc', - 'src/core/lib/security/context/security_context.cc', - 'src/core/lib/security/credentials/alts/alts_credentials.cc', - 'src/core/lib/security/credentials/alts/check_gcp_environment.cc', - 'src/core/lib/security/credentials/alts/check_gcp_environment_linux.cc', - 'src/core/lib/security/credentials/alts/check_gcp_environment_no_op.cc', - 'src/core/lib/security/credentials/alts/check_gcp_environment_windows.cc', - 'src/core/lib/security/credentials/alts/grpc_alts_credentials_client_options.cc', - 'src/core/lib/security/credentials/alts/grpc_alts_credentials_options.cc', - 'src/core/lib/security/credentials/alts/grpc_alts_credentials_server_options.cc', - 'src/core/lib/security/credentials/call_creds_util.cc', - 'src/core/lib/security/credentials/channel_creds_registry_init.cc', - 'src/core/lib/security/credentials/composite/composite_credentials.cc', - 'src/core/lib/security/credentials/credentials.cc', - 'src/core/lib/security/credentials/external/aws_external_account_credentials.cc', - 'src/core/lib/security/credentials/external/aws_request_signer.cc', - 'src/core/lib/security/credentials/external/external_account_credentials.cc', - 'src/core/lib/security/credentials/external/file_external_account_credentials.cc', - 'src/core/lib/security/credentials/external/url_external_account_credentials.cc', - 'src/core/lib/security/credentials/fake/fake_credentials.cc', - 'src/core/lib/security/credentials/google_default/credentials_generic.cc', - 'src/core/lib/security/credentials/google_default/google_default_credentials.cc', - 'src/core/lib/security/credentials/iam/iam_credentials.cc', - 'src/core/lib/security/credentials/insecure/insecure_credentials.cc', - 'src/core/lib/security/credentials/jwt/json_token.cc', - 'src/core/lib/security/credentials/jwt/jwt_credentials.cc', - 'src/core/lib/security/credentials/jwt/jwt_verifier.cc', - 'src/core/lib/security/credentials/local/local_credentials.cc', - 'src/core/lib/security/credentials/oauth2/oauth2_credentials.cc', - 'src/core/lib/security/credentials/plugin/plugin_credentials.cc', - 'src/core/lib/security/credentials/ssl/ssl_credentials.cc', - 'src/core/lib/security/credentials/tls/grpc_tls_certificate_distributor.cc', - 'src/core/lib/security/credentials/tls/grpc_tls_certificate_match.cc', - 'src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.cc', - 'src/core/lib/security/credentials/tls/grpc_tls_certificate_verifier.cc', - 'src/core/lib/security/credentials/tls/grpc_tls_credentials_options.cc', - 'src/core/lib/security/credentials/tls/grpc_tls_crl_provider.cc', - 'src/core/lib/security/credentials/tls/tls_credentials.cc', - 'src/core/lib/security/credentials/tls/tls_utils.cc', - 'src/core/lib/security/credentials/xds/xds_credentials.cc', - 'src/core/lib/security/security_connector/alts/alts_security_connector.cc', - 'src/core/lib/security/security_connector/fake/fake_security_connector.cc', - 'src/core/lib/security/security_connector/insecure/insecure_security_connector.cc', - 'src/core/lib/security/security_connector/load_system_roots_fallback.cc', - 'src/core/lib/security/security_connector/load_system_roots_supported.cc', - 'src/core/lib/security/security_connector/load_system_roots_windows.cc', - 'src/core/lib/security/security_connector/local/local_security_connector.cc', - 'src/core/lib/security/security_connector/security_connector.cc', - 'src/core/lib/security/security_connector/ssl/ssl_security_connector.cc', - 'src/core/lib/security/security_connector/ssl_utils.cc', - 'src/core/lib/security/security_connector/tls/tls_security_connector.cc', - 'src/core/lib/security/transport/client_auth_filter.cc', - 'src/core/lib/security/transport/legacy_server_auth_filter.cc', - 'src/core/lib/security/transport/secure_endpoint.cc', - 'src/core/lib/security/transport/security_handshaker.cc', - 'src/core/lib/security/transport/server_auth_filter.cc', - 'src/core/lib/security/transport/tsi_error.cc', - 'src/core/lib/security/util/json_util.cc', - 'src/core/lib/slice/b64.cc', - 'src/core/lib/slice/percent_encoding.cc', - 'src/core/lib/slice/slice.cc', - 'src/core/lib/slice/slice_buffer.cc', - 'src/core/lib/slice/slice_refcount.cc', - 'src/core/lib/slice/slice_string_helpers.cc', - 'src/core/lib/surface/api_trace.cc', - 'src/core/lib/surface/byte_buffer.cc', - 'src/core/lib/surface/byte_buffer_reader.cc', - 'src/core/lib/surface/call.cc', - 'src/core/lib/surface/call_details.cc', - 'src/core/lib/surface/call_log_batch.cc', - 'src/core/lib/surface/channel.cc', - 'src/core/lib/surface/channel_create.cc', - 'src/core/lib/surface/channel_init.cc', - 'src/core/lib/surface/channel_stack_type.cc', - 'src/core/lib/surface/completion_queue.cc', - 'src/core/lib/surface/completion_queue_factory.cc', - 'src/core/lib/surface/event_string.cc', - 'src/core/lib/surface/init.cc', - 'src/core/lib/surface/init_internally.cc', - 'src/core/lib/surface/lame_client.cc', - 'src/core/lib/surface/legacy_channel.cc', - 'src/core/lib/surface/metadata_array.cc', - 'src/core/lib/surface/server.cc', - 'src/core/lib/surface/validate_metadata.cc', - 'src/core/lib/surface/version.cc', - 'src/core/lib/surface/wait_for_cq_end_op.cc', - 'src/core/lib/transport/batch_builder.cc', - 'src/core/lib/transport/bdp_estimator.cc', - 'src/core/lib/transport/call_factory.cc', - 'src/core/lib/transport/call_filters.cc', - 'src/core/lib/transport/call_final_info.cc', - 'src/core/lib/transport/call_size_estimator.cc', - 'src/core/lib/transport/call_spine.cc', - 'src/core/lib/transport/connectivity_state.cc', - 'src/core/lib/transport/error_utils.cc', - 'src/core/lib/transport/handshaker.cc', - 'src/core/lib/transport/handshaker_registry.cc', - 'src/core/lib/transport/http_connect_handshaker.cc', - 'src/core/lib/transport/message.cc', - 'src/core/lib/transport/metadata.cc', - 'src/core/lib/transport/metadata_batch.cc', - 'src/core/lib/transport/metadata_info.cc', - 'src/core/lib/transport/parsed_metadata.cc', - 'src/core/lib/transport/status_conversion.cc', - 'src/core/lib/transport/tcp_connect_handshaker.cc', - 'src/core/lib/transport/timeout_encoding.cc', - 'src/core/lib/transport/transport.cc', - 'src/core/lib/transport/transport_op_string.cc', - 'src/core/lib/uri/uri_parser.cc', - 'src/core/load_balancing/address_filtering.cc', - 'src/core/load_balancing/backend_metric_parser.cc', - 'src/core/load_balancing/child_policy_handler.cc', - 'src/core/load_balancing/endpoint_list.cc', - 'src/core/load_balancing/grpclb/client_load_reporting_filter.cc', - 'src/core/load_balancing/grpclb/grpclb.cc', - 'src/core/load_balancing/grpclb/grpclb_balancer_addresses.cc', - 'src/core/load_balancing/grpclb/grpclb_client_stats.cc', - 'src/core/load_balancing/grpclb/load_balancer_api.cc', - 'src/core/load_balancing/health_check_client.cc', - 'src/core/load_balancing/lb_policy.cc', - 'src/core/load_balancing/lb_policy_registry.cc', - 'src/core/load_balancing/oob_backend_metric.cc', - 'src/core/load_balancing/outlier_detection/outlier_detection.cc', - 'src/core/load_balancing/pick_first/pick_first.cc', - 'src/core/load_balancing/priority/priority.cc', - 'src/core/load_balancing/ring_hash/ring_hash.cc', - 'src/core/load_balancing/rls/rls.cc', - 'src/core/load_balancing/round_robin/round_robin.cc', - 'src/core/load_balancing/weighted_round_robin/static_stride_scheduler.cc', - 'src/core/load_balancing/weighted_round_robin/weighted_round_robin.cc', - 'src/core/load_balancing/weighted_target/weighted_target.cc', - 'src/core/load_balancing/xds/cds.cc', - 'src/core/load_balancing/xds/xds_cluster_impl.cc', - 'src/core/load_balancing/xds/xds_cluster_manager.cc', - 'src/core/load_balancing/xds/xds_override_host.cc', - 'src/core/load_balancing/xds/xds_wrr_locality.cc', - 'src/core/plugin_registry/grpc_plugin_registry.cc', - 'src/core/plugin_registry/grpc_plugin_registry_extra.cc', - 'src/core/resolver/binder/binder_resolver.cc', - 'src/core/resolver/dns/c_ares/dns_resolver_ares.cc', - 'src/core/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc', - 'src/core/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc', - 'src/core/resolver/dns/c_ares/grpc_ares_wrapper.cc', - 'src/core/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc', - 'src/core/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc', - 'src/core/resolver/dns/dns_resolver_plugin.cc', - 'src/core/resolver/dns/event_engine/event_engine_client_channel_resolver.cc', - 'src/core/resolver/dns/event_engine/service_config_helper.cc', - 'src/core/resolver/dns/native/dns_resolver.cc', - 'src/core/resolver/endpoint_addresses.cc', - 'src/core/resolver/fake/fake_resolver.cc', - 'src/core/resolver/google_c2p/google_c2p_resolver.cc', - 'src/core/resolver/polling_resolver.cc', - 'src/core/resolver/resolver.cc', - 'src/core/resolver/resolver_registry.cc', - 'src/core/resolver/sockaddr/sockaddr_resolver.cc', - 'src/core/resolver/xds/xds_dependency_manager.cc', - 'src/core/resolver/xds/xds_resolver.cc', - 'src/core/resolver/xds/xds_resolver_trace.cc', - 'src/core/service_config/service_config_channel_arg_filter.cc', - 'src/core/service_config/service_config_impl.cc', - 'src/core/service_config/service_config_parser.cc', - 'src/core/tsi/alts/crypt/aes_gcm.cc', - 'src/core/tsi/alts/crypt/gsec.cc', - 'src/core/tsi/alts/frame_protector/alts_counter.cc', - 'src/core/tsi/alts/frame_protector/alts_crypter.cc', - 'src/core/tsi/alts/frame_protector/alts_frame_protector.cc', - 'src/core/tsi/alts/frame_protector/alts_record_protocol_crypter_common.cc', - 'src/core/tsi/alts/frame_protector/alts_seal_privacy_integrity_crypter.cc', - 'src/core/tsi/alts/frame_protector/alts_unseal_privacy_integrity_crypter.cc', - 'src/core/tsi/alts/frame_protector/frame_handler.cc', - 'src/core/tsi/alts/handshaker/alts_handshaker_client.cc', - 'src/core/tsi/alts/handshaker/alts_shared_resource.cc', - 'src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc', - 'src/core/tsi/alts/handshaker/alts_tsi_utils.cc', - 'src/core/tsi/alts/handshaker/transport_security_common_api.cc', - 'src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_integrity_only_record_protocol.cc', - 'src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_privacy_integrity_record_protocol.cc', - 'src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_record_protocol_common.cc', - 'src/core/tsi/alts/zero_copy_frame_protector/alts_iovec_record_protocol.cc', - 'src/core/tsi/alts/zero_copy_frame_protector/alts_zero_copy_grpc_protector.cc', - 'src/core/tsi/fake_transport_security.cc', - 'src/core/tsi/local_transport_security.cc', - 'src/core/tsi/ssl/key_logging/ssl_key_logging.cc', - 'src/core/tsi/ssl/session_cache/ssl_session_boringssl.cc', - 'src/core/tsi/ssl/session_cache/ssl_session_cache.cc', - 'src/core/tsi/ssl/session_cache/ssl_session_openssl.cc', - 'src/core/tsi/ssl_transport_security.cc', - 'src/core/tsi/ssl_transport_security_utils.cc', - 'src/core/tsi/transport_security.cc', - 'src/core/tsi/transport_security_grpc.cc', - ], - }, - { - 'target_name': 'grpc_test_util', - 'type': 'static_library', - 'dependencies': [ - 'absl/debugging:failure_signal_handler', - 'absl/debugging:stacktrace', - 'absl/debugging:symbolize', - 'grpc', - ], - 'sources': [ - 'test/core/event_engine/test_init.cc', - 'test/core/util/build.cc', - 'test/core/util/port.cc', - 'test/core/util/port_isolated_runtime_environment.cc', - 'test/core/util/port_server_client.cc', - 'test/core/util/reconnect_server.cc', - 'test/core/util/stack_tracer.cc', - 'test/core/util/test_config.cc', - 'test/core/util/test_tcp_server.cc', - 'test/core/util/tls_utils.cc', - ], - }, - { - 'target_name': 'grpc_test_util_unsecure', - 'type': 'static_library', - 'dependencies': [ - 'absl/debugging:failure_signal_handler', - 'absl/debugging:stacktrace', - 'absl/debugging:symbolize', - 'grpc_unsecure', - ], - 'sources': [ - 'test/core/event_engine/test_init.cc', - 'test/core/util/build.cc', - 'test/core/util/port.cc', - 'test/core/util/port_isolated_runtime_environment.cc', - 'test/core/util/port_server_client.cc', - 'test/core/util/reconnect_server.cc', - 'test/core/util/stack_tracer.cc', - 'test/core/util/test_config.cc', - 'test/core/util/test_tcp_server.cc', - ], - }, - { - 'target_name': 'grpc_unsecure', - 'type': 'static_library', - 'dependencies': [ - 'upb_message_lib', - 'utf8_range_lib', - 'z', - 'absl/algorithm:container', - 'absl/base:config', - 'absl/base:no_destructor', - 'absl/cleanup:cleanup', - 'absl/container:flat_hash_map', - 'absl/container:flat_hash_set', - 'absl/container:inlined_vector', - 'absl/functional:bind_front', - 'absl/functional:function_ref', - 'absl/hash:hash', - 'absl/meta:type_traits', - 'absl/random:bit_gen_ref', - 'absl/random:distributions', - 'absl/status:statusor', - 'absl/types:span', - 'absl/utility:utility', - 'cares', - 'gpr', - 'address_sorting', - ], - 'sources': [ - 'src/core/client_channel/backup_poller.cc', - 'src/core/client_channel/client_channel_channelz.cc', - 'src/core/client_channel/client_channel_factory.cc', - 'src/core/client_channel/client_channel_filter.cc', - 'src/core/client_channel/client_channel_plugin.cc', - 'src/core/client_channel/client_channel_service_config.cc', - 'src/core/client_channel/config_selector.cc', - 'src/core/client_channel/dynamic_filters.cc', - 'src/core/client_channel/global_subchannel_pool.cc', - 'src/core/client_channel/http_proxy_mapper.cc', - 'src/core/client_channel/local_subchannel_pool.cc', - 'src/core/client_channel/retry_filter.cc', - 'src/core/client_channel/retry_filter_legacy_call_data.cc', - 'src/core/client_channel/retry_service_config.cc', - 'src/core/client_channel/retry_throttle.cc', - 'src/core/client_channel/subchannel.cc', - 'src/core/client_channel/subchannel_pool_interface.cc', - 'src/core/client_channel/subchannel_stream_client.cc', - 'src/core/ext/filters/backend_metrics/backend_metric_filter.cc', - 'src/core/ext/filters/census/grpc_context.cc', - 'src/core/ext/filters/channel_idle/channel_idle_filter.cc', - 'src/core/ext/filters/channel_idle/idle_filter_state.cc', - 'src/core/ext/filters/channel_idle/legacy_channel_idle_filter.cc', - 'src/core/ext/filters/deadline/deadline_filter.cc', - 'src/core/ext/filters/fault_injection/fault_injection_filter.cc', - 'src/core/ext/filters/fault_injection/fault_injection_service_config_parser.cc', - 'src/core/ext/filters/http/client/http_client_filter.cc', - 'src/core/ext/filters/http/client_authority_filter.cc', - 'src/core/ext/filters/http/http_filters_plugin.cc', - 'src/core/ext/filters/http/message_compress/compression_filter.cc', - 'src/core/ext/filters/http/message_compress/legacy_compression_filter.cc', - 'src/core/ext/filters/http/server/http_server_filter.cc', - 'src/core/ext/filters/message_size/message_size_filter.cc', - 'src/core/ext/transport/chttp2/client/chttp2_connector.cc', - 'src/core/ext/transport/chttp2/server/chttp2_server.cc', - 'src/core/ext/transport/chttp2/transport/bin_decoder.cc', - 'src/core/ext/transport/chttp2/transport/bin_encoder.cc', - 'src/core/ext/transport/chttp2/transport/chttp2_transport.cc', - 'src/core/ext/transport/chttp2/transport/decode_huff.cc', - 'src/core/ext/transport/chttp2/transport/flow_control.cc', - 'src/core/ext/transport/chttp2/transport/frame.cc', - 'src/core/ext/transport/chttp2/transport/frame_data.cc', - 'src/core/ext/transport/chttp2/transport/frame_goaway.cc', - 'src/core/ext/transport/chttp2/transport/frame_ping.cc', - 'src/core/ext/transport/chttp2/transport/frame_rst_stream.cc', - 'src/core/ext/transport/chttp2/transport/frame_settings.cc', - 'src/core/ext/transport/chttp2/transport/frame_window_update.cc', - 'src/core/ext/transport/chttp2/transport/hpack_encoder.cc', - 'src/core/ext/transport/chttp2/transport/hpack_encoder_table.cc', - 'src/core/ext/transport/chttp2/transport/hpack_parse_result.cc', - 'src/core/ext/transport/chttp2/transport/hpack_parser.cc', - 'src/core/ext/transport/chttp2/transport/hpack_parser_table.cc', - 'src/core/ext/transport/chttp2/transport/http2_settings.cc', - 'src/core/ext/transport/chttp2/transport/http_trace.cc', - 'src/core/ext/transport/chttp2/transport/huffsyms.cc', - 'src/core/ext/transport/chttp2/transport/max_concurrent_streams_policy.cc', - 'src/core/ext/transport/chttp2/transport/parsing.cc', - 'src/core/ext/transport/chttp2/transport/ping_abuse_policy.cc', - 'src/core/ext/transport/chttp2/transport/ping_callbacks.cc', - 'src/core/ext/transport/chttp2/transport/ping_rate_policy.cc', - 'src/core/ext/transport/chttp2/transport/stream_lists.cc', - 'src/core/ext/transport/chttp2/transport/varint.cc', - 'src/core/ext/transport/chttp2/transport/write_size_policy.cc', - 'src/core/ext/transport/chttp2/transport/writing.cc', - 'src/core/ext/transport/inproc/inproc_plugin.cc', - 'src/core/ext/transport/inproc/inproc_transport.cc', - 'src/core/ext/transport/inproc/legacy_inproc_transport.cc', - 'src/core/ext/upb-gen/google/api/annotations.upb_minitable.c', - 'src/core/ext/upb-gen/google/api/http.upb_minitable.c', - 'src/core/ext/upb-gen/google/protobuf/any.upb_minitable.c', - 'src/core/ext/upb-gen/google/protobuf/descriptor.upb_minitable.c', - 'src/core/ext/upb-gen/google/protobuf/duration.upb_minitable.c', - 'src/core/ext/upb-gen/google/protobuf/empty.upb_minitable.c', - 'src/core/ext/upb-gen/google/protobuf/struct.upb_minitable.c', - 'src/core/ext/upb-gen/google/protobuf/timestamp.upb_minitable.c', - 'src/core/ext/upb-gen/google/protobuf/wrappers.upb_minitable.c', - 'src/core/ext/upb-gen/google/rpc/status.upb_minitable.c', - 'src/core/ext/upb-gen/src/proto/grpc/gcp/altscontext.upb_minitable.c', - 'src/core/ext/upb-gen/src/proto/grpc/gcp/handshaker.upb_minitable.c', - 'src/core/ext/upb-gen/src/proto/grpc/gcp/transport_security_common.upb_minitable.c', - 'src/core/ext/upb-gen/src/proto/grpc/health/v1/health.upb_minitable.c', - 'src/core/ext/upb-gen/src/proto/grpc/lb/v1/load_balancer.upb_minitable.c', - 'src/core/ext/upb-gen/src/proto/grpc/lookup/v1/rls.upb_minitable.c', - 'src/core/ext/upb-gen/validate/validate.upb_minitable.c', - 'src/core/ext/upb-gen/xds/data/orca/v3/orca_load_report.upb_minitable.c', - 'src/core/ext/upb-gen/xds/service/orca/v3/orca.upb_minitable.c', - 'src/core/lib/address_utils/parse_address.cc', - 'src/core/lib/address_utils/sockaddr_utils.cc', - 'src/core/lib/backoff/backoff.cc', - 'src/core/lib/backoff/random_early_detection.cc', - 'src/core/lib/channel/call_tracer.cc', - 'src/core/lib/channel/channel_args.cc', - 'src/core/lib/channel/channel_args_preconditioning.cc', - 'src/core/lib/channel/channel_stack.cc', - 'src/core/lib/channel/channel_stack_builder.cc', - 'src/core/lib/channel/channel_stack_builder_impl.cc', - 'src/core/lib/channel/channel_stack_trace.cc', - 'src/core/lib/channel/channel_trace.cc', - 'src/core/lib/channel/channelz.cc', - 'src/core/lib/channel/channelz_registry.cc', - 'src/core/lib/channel/connected_channel.cc', - 'src/core/lib/channel/metrics.cc', - 'src/core/lib/channel/promise_based_filter.cc', - 'src/core/lib/channel/server_call_tracer_filter.cc', - 'src/core/lib/channel/status_util.cc', - 'src/core/lib/compression/compression.cc', - 'src/core/lib/compression/compression_internal.cc', - 'src/core/lib/compression/message_compress.cc', - 'src/core/lib/config/core_configuration.cc', - 'src/core/lib/debug/event_log.cc', - 'src/core/lib/debug/histogram_view.cc', - 'src/core/lib/debug/stats.cc', - 'src/core/lib/debug/stats_data.cc', - 'src/core/lib/debug/trace.cc', - 'src/core/lib/event_engine/ares_resolver.cc', - 'src/core/lib/event_engine/cf_engine/cf_engine.cc', - 'src/core/lib/event_engine/cf_engine/cfstream_endpoint.cc', - 'src/core/lib/event_engine/cf_engine/dns_service_resolver.cc', - 'src/core/lib/event_engine/channel_args_endpoint_config.cc', - 'src/core/lib/event_engine/default_event_engine.cc', - 'src/core/lib/event_engine/default_event_engine_factory.cc', - 'src/core/lib/event_engine/event_engine.cc', - 'src/core/lib/event_engine/forkable.cc', - 'src/core/lib/event_engine/posix_engine/ev_epoll1_linux.cc', - 'src/core/lib/event_engine/posix_engine/ev_poll_posix.cc', - 'src/core/lib/event_engine/posix_engine/event_poller_posix_default.cc', - 'src/core/lib/event_engine/posix_engine/internal_errqueue.cc', - 'src/core/lib/event_engine/posix_engine/lockfree_event.cc', - 'src/core/lib/event_engine/posix_engine/native_posix_dns_resolver.cc', - 'src/core/lib/event_engine/posix_engine/posix_endpoint.cc', - 'src/core/lib/event_engine/posix_engine/posix_engine.cc', - 'src/core/lib/event_engine/posix_engine/posix_engine_listener.cc', - 'src/core/lib/event_engine/posix_engine/posix_engine_listener_utils.cc', - 'src/core/lib/event_engine/posix_engine/tcp_socket_utils.cc', - 'src/core/lib/event_engine/posix_engine/timer.cc', - 'src/core/lib/event_engine/posix_engine/timer_heap.cc', - 'src/core/lib/event_engine/posix_engine/timer_manager.cc', - 'src/core/lib/event_engine/posix_engine/traced_buffer_list.cc', - 'src/core/lib/event_engine/posix_engine/wakeup_fd_eventfd.cc', - 'src/core/lib/event_engine/posix_engine/wakeup_fd_pipe.cc', - 'src/core/lib/event_engine/posix_engine/wakeup_fd_posix_default.cc', - 'src/core/lib/event_engine/resolved_address.cc', - 'src/core/lib/event_engine/shim.cc', - 'src/core/lib/event_engine/slice.cc', - 'src/core/lib/event_engine/slice_buffer.cc', - 'src/core/lib/event_engine/tcp_socket_utils.cc', - 'src/core/lib/event_engine/thread_pool/thread_count.cc', - 'src/core/lib/event_engine/thread_pool/thread_pool_factory.cc', - 'src/core/lib/event_engine/thread_pool/work_stealing_thread_pool.cc', - 'src/core/lib/event_engine/thready_event_engine/thready_event_engine.cc', - 'src/core/lib/event_engine/time_util.cc', - 'src/core/lib/event_engine/trace.cc', - 'src/core/lib/event_engine/utils.cc', - 'src/core/lib/event_engine/windows/grpc_polled_fd_windows.cc', - 'src/core/lib/event_engine/windows/iocp.cc', - 'src/core/lib/event_engine/windows/native_windows_dns_resolver.cc', - 'src/core/lib/event_engine/windows/win_socket.cc', - 'src/core/lib/event_engine/windows/windows_endpoint.cc', - 'src/core/lib/event_engine/windows/windows_engine.cc', - 'src/core/lib/event_engine/windows/windows_listener.cc', - 'src/core/lib/event_engine/work_queue/basic_work_queue.cc', - 'src/core/lib/experiments/config.cc', - 'src/core/lib/experiments/experiments.cc', - 'src/core/lib/gprpp/load_file.cc', - 'src/core/lib/gprpp/per_cpu.cc', - 'src/core/lib/gprpp/ref_counted_string.cc', - 'src/core/lib/gprpp/status_helper.cc', - 'src/core/lib/gprpp/time.cc', - 'src/core/lib/gprpp/time_averaged_stats.cc', - 'src/core/lib/gprpp/uuid_v4.cc', - 'src/core/lib/gprpp/validation_errors.cc', - 'src/core/lib/gprpp/work_serializer.cc', - 'src/core/lib/handshaker/proxy_mapper_registry.cc', - 'src/core/lib/http/format_request.cc', - 'src/core/lib/http/httpcli.cc', - 'src/core/lib/http/parser.cc', - 'src/core/lib/iomgr/buffer_list.cc', - 'src/core/lib/iomgr/call_combiner.cc', - 'src/core/lib/iomgr/cfstream_handle.cc', - 'src/core/lib/iomgr/closure.cc', - 'src/core/lib/iomgr/combiner.cc', - 'src/core/lib/iomgr/dualstack_socket_posix.cc', - 'src/core/lib/iomgr/endpoint.cc', - 'src/core/lib/iomgr/endpoint_cfstream.cc', - 'src/core/lib/iomgr/endpoint_pair_posix.cc', - 'src/core/lib/iomgr/endpoint_pair_windows.cc', - 'src/core/lib/iomgr/error.cc', - 'src/core/lib/iomgr/error_cfstream.cc', - 'src/core/lib/iomgr/ev_apple.cc', - 'src/core/lib/iomgr/ev_epoll1_linux.cc', - 'src/core/lib/iomgr/ev_poll_posix.cc', - 'src/core/lib/iomgr/ev_posix.cc', - 'src/core/lib/iomgr/ev_windows.cc', - 'src/core/lib/iomgr/event_engine_shims/closure.cc', - 'src/core/lib/iomgr/event_engine_shims/endpoint.cc', - 'src/core/lib/iomgr/event_engine_shims/tcp_client.cc', - 'src/core/lib/iomgr/exec_ctx.cc', - 'src/core/lib/iomgr/executor.cc', - 'src/core/lib/iomgr/fork_posix.cc', - 'src/core/lib/iomgr/fork_windows.cc', - 'src/core/lib/iomgr/gethostname_fallback.cc', - 'src/core/lib/iomgr/gethostname_host_name_max.cc', - 'src/core/lib/iomgr/gethostname_sysconf.cc', - 'src/core/lib/iomgr/grpc_if_nametoindex_posix.cc', - 'src/core/lib/iomgr/grpc_if_nametoindex_unsupported.cc', - 'src/core/lib/iomgr/internal_errqueue.cc', - 'src/core/lib/iomgr/iocp_windows.cc', - 'src/core/lib/iomgr/iomgr.cc', - 'src/core/lib/iomgr/iomgr_internal.cc', - 'src/core/lib/iomgr/iomgr_posix.cc', - 'src/core/lib/iomgr/iomgr_posix_cfstream.cc', - 'src/core/lib/iomgr/iomgr_windows.cc', - 'src/core/lib/iomgr/lockfree_event.cc', - 'src/core/lib/iomgr/polling_entity.cc', - 'src/core/lib/iomgr/pollset.cc', - 'src/core/lib/iomgr/pollset_set.cc', - 'src/core/lib/iomgr/pollset_set_windows.cc', - 'src/core/lib/iomgr/pollset_windows.cc', - 'src/core/lib/iomgr/resolve_address.cc', - 'src/core/lib/iomgr/resolve_address_posix.cc', - 'src/core/lib/iomgr/resolve_address_windows.cc', - 'src/core/lib/iomgr/sockaddr_utils_posix.cc', - 'src/core/lib/iomgr/socket_factory_posix.cc', - 'src/core/lib/iomgr/socket_mutator.cc', - 'src/core/lib/iomgr/socket_utils_common_posix.cc', - 'src/core/lib/iomgr/socket_utils_linux.cc', - 'src/core/lib/iomgr/socket_utils_posix.cc', - 'src/core/lib/iomgr/socket_utils_windows.cc', - 'src/core/lib/iomgr/socket_windows.cc', - 'src/core/lib/iomgr/systemd_utils.cc', - 'src/core/lib/iomgr/tcp_client.cc', - 'src/core/lib/iomgr/tcp_client_cfstream.cc', - 'src/core/lib/iomgr/tcp_client_posix.cc', - 'src/core/lib/iomgr/tcp_client_windows.cc', - 'src/core/lib/iomgr/tcp_posix.cc', - 'src/core/lib/iomgr/tcp_server.cc', - 'src/core/lib/iomgr/tcp_server_posix.cc', - 'src/core/lib/iomgr/tcp_server_utils_posix_common.cc', - 'src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.cc', - 'src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.cc', - 'src/core/lib/iomgr/tcp_server_windows.cc', - 'src/core/lib/iomgr/tcp_windows.cc', - 'src/core/lib/iomgr/timer.cc', - 'src/core/lib/iomgr/timer_generic.cc', - 'src/core/lib/iomgr/timer_heap.cc', - 'src/core/lib/iomgr/timer_manager.cc', - 'src/core/lib/iomgr/unix_sockets_posix.cc', - 'src/core/lib/iomgr/unix_sockets_posix_noop.cc', - 'src/core/lib/iomgr/vsock.cc', - 'src/core/lib/iomgr/wakeup_fd_eventfd.cc', - 'src/core/lib/iomgr/wakeup_fd_nospecial.cc', - 'src/core/lib/iomgr/wakeup_fd_pipe.cc', - 'src/core/lib/iomgr/wakeup_fd_posix.cc', - 'src/core/lib/json/json_object_loader.cc', - 'src/core/lib/json/json_reader.cc', - 'src/core/lib/json/json_writer.cc', - 'src/core/lib/promise/activity.cc', - 'src/core/lib/promise/party.cc', - 'src/core/lib/promise/sleep.cc', - 'src/core/lib/promise/trace.cc', - 'src/core/lib/resource_quota/api.cc', - 'src/core/lib/resource_quota/arena.cc', - 'src/core/lib/resource_quota/connection_quota.cc', - 'src/core/lib/resource_quota/memory_quota.cc', - 'src/core/lib/resource_quota/periodic_update.cc', - 'src/core/lib/resource_quota/resource_quota.cc', - 'src/core/lib/resource_quota/thread_quota.cc', - 'src/core/lib/resource_quota/trace.cc', - 'src/core/lib/security/authorization/authorization_policy_provider_vtable.cc', - 'src/core/lib/security/authorization/evaluate_args.cc', - 'src/core/lib/security/authorization/grpc_server_authz_filter.cc', - 'src/core/lib/security/certificate_provider/certificate_provider_registry.cc', - 'src/core/lib/security/context/security_context.cc', - 'src/core/lib/security/credentials/alts/check_gcp_environment.cc', - 'src/core/lib/security/credentials/alts/check_gcp_environment_linux.cc', - 'src/core/lib/security/credentials/alts/check_gcp_environment_no_op.cc', - 'src/core/lib/security/credentials/alts/check_gcp_environment_windows.cc', - 'src/core/lib/security/credentials/alts/grpc_alts_credentials_client_options.cc', - 'src/core/lib/security/credentials/alts/grpc_alts_credentials_options.cc', - 'src/core/lib/security/credentials/alts/grpc_alts_credentials_server_options.cc', - 'src/core/lib/security/credentials/call_creds_util.cc', - 'src/core/lib/security/credentials/composite/composite_credentials.cc', - 'src/core/lib/security/credentials/credentials.cc', - 'src/core/lib/security/credentials/fake/fake_credentials.cc', - 'src/core/lib/security/credentials/insecure/insecure_credentials.cc', - 'src/core/lib/security/credentials/plugin/plugin_credentials.cc', - 'src/core/lib/security/credentials/tls/tls_utils.cc', - 'src/core/lib/security/security_connector/fake/fake_security_connector.cc', - 'src/core/lib/security/security_connector/insecure/insecure_security_connector.cc', - 'src/core/lib/security/security_connector/load_system_roots_fallback.cc', - 'src/core/lib/security/security_connector/load_system_roots_supported.cc', - 'src/core/lib/security/security_connector/load_system_roots_windows.cc', - 'src/core/lib/security/security_connector/security_connector.cc', - 'src/core/lib/security/transport/client_auth_filter.cc', - 'src/core/lib/security/transport/legacy_server_auth_filter.cc', - 'src/core/lib/security/transport/secure_endpoint.cc', - 'src/core/lib/security/transport/security_handshaker.cc', - 'src/core/lib/security/transport/server_auth_filter.cc', - 'src/core/lib/security/transport/tsi_error.cc', - 'src/core/lib/security/util/json_util.cc', - 'src/core/lib/slice/b64.cc', - 'src/core/lib/slice/percent_encoding.cc', - 'src/core/lib/slice/slice.cc', - 'src/core/lib/slice/slice_buffer.cc', - 'src/core/lib/slice/slice_refcount.cc', - 'src/core/lib/slice/slice_string_helpers.cc', - 'src/core/lib/surface/api_trace.cc', - 'src/core/lib/surface/byte_buffer.cc', - 'src/core/lib/surface/byte_buffer_reader.cc', - 'src/core/lib/surface/call.cc', - 'src/core/lib/surface/call_details.cc', - 'src/core/lib/surface/call_log_batch.cc', - 'src/core/lib/surface/channel.cc', - 'src/core/lib/surface/channel_create.cc', - 'src/core/lib/surface/channel_init.cc', - 'src/core/lib/surface/channel_stack_type.cc', - 'src/core/lib/surface/completion_queue.cc', - 'src/core/lib/surface/completion_queue_factory.cc', - 'src/core/lib/surface/event_string.cc', - 'src/core/lib/surface/init.cc', - 'src/core/lib/surface/init_internally.cc', - 'src/core/lib/surface/lame_client.cc', - 'src/core/lib/surface/legacy_channel.cc', - 'src/core/lib/surface/metadata_array.cc', - 'src/core/lib/surface/server.cc', - 'src/core/lib/surface/validate_metadata.cc', - 'src/core/lib/surface/version.cc', - 'src/core/lib/surface/wait_for_cq_end_op.cc', - 'src/core/lib/transport/batch_builder.cc', - 'src/core/lib/transport/bdp_estimator.cc', - 'src/core/lib/transport/call_factory.cc', - 'src/core/lib/transport/call_filters.cc', - 'src/core/lib/transport/call_final_info.cc', - 'src/core/lib/transport/call_size_estimator.cc', - 'src/core/lib/transport/call_spine.cc', - 'src/core/lib/transport/connectivity_state.cc', - 'src/core/lib/transport/error_utils.cc', - 'src/core/lib/transport/handshaker.cc', - 'src/core/lib/transport/handshaker_registry.cc', - 'src/core/lib/transport/http_connect_handshaker.cc', - 'src/core/lib/transport/message.cc', - 'src/core/lib/transport/metadata.cc', - 'src/core/lib/transport/metadata_batch.cc', - 'src/core/lib/transport/metadata_info.cc', - 'src/core/lib/transport/parsed_metadata.cc', - 'src/core/lib/transport/status_conversion.cc', - 'src/core/lib/transport/tcp_connect_handshaker.cc', - 'src/core/lib/transport/timeout_encoding.cc', - 'src/core/lib/transport/transport.cc', - 'src/core/lib/transport/transport_op_string.cc', - 'src/core/lib/uri/uri_parser.cc', - 'src/core/load_balancing/address_filtering.cc', - 'src/core/load_balancing/backend_metric_parser.cc', - 'src/core/load_balancing/child_policy_handler.cc', - 'src/core/load_balancing/endpoint_list.cc', - 'src/core/load_balancing/grpclb/client_load_reporting_filter.cc', - 'src/core/load_balancing/grpclb/grpclb.cc', - 'src/core/load_balancing/grpclb/grpclb_balancer_addresses.cc', - 'src/core/load_balancing/grpclb/grpclb_client_stats.cc', - 'src/core/load_balancing/grpclb/load_balancer_api.cc', - 'src/core/load_balancing/health_check_client.cc', - 'src/core/load_balancing/lb_policy.cc', - 'src/core/load_balancing/lb_policy_registry.cc', - 'src/core/load_balancing/oob_backend_metric.cc', - 'src/core/load_balancing/outlier_detection/outlier_detection.cc', - 'src/core/load_balancing/pick_first/pick_first.cc', - 'src/core/load_balancing/priority/priority.cc', - 'src/core/load_balancing/rls/rls.cc', - 'src/core/load_balancing/round_robin/round_robin.cc', - 'src/core/load_balancing/weighted_round_robin/static_stride_scheduler.cc', - 'src/core/load_balancing/weighted_round_robin/weighted_round_robin.cc', - 'src/core/load_balancing/weighted_target/weighted_target.cc', - 'src/core/plugin_registry/grpc_plugin_registry.cc', - 'src/core/plugin_registry/grpc_plugin_registry_noextra.cc', - 'src/core/resolver/binder/binder_resolver.cc', - 'src/core/resolver/dns/c_ares/dns_resolver_ares.cc', - 'src/core/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc', - 'src/core/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc', - 'src/core/resolver/dns/c_ares/grpc_ares_wrapper.cc', - 'src/core/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc', - 'src/core/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc', - 'src/core/resolver/dns/dns_resolver_plugin.cc', - 'src/core/resolver/dns/event_engine/event_engine_client_channel_resolver.cc', - 'src/core/resolver/dns/event_engine/service_config_helper.cc', - 'src/core/resolver/dns/native/dns_resolver.cc', - 'src/core/resolver/endpoint_addresses.cc', - 'src/core/resolver/fake/fake_resolver.cc', - 'src/core/resolver/polling_resolver.cc', - 'src/core/resolver/resolver.cc', - 'src/core/resolver/resolver_registry.cc', - 'src/core/resolver/sockaddr/sockaddr_resolver.cc', - 'src/core/service_config/service_config_channel_arg_filter.cc', - 'src/core/service_config/service_config_impl.cc', - 'src/core/service_config/service_config_parser.cc', - 'src/core/tsi/alts/handshaker/transport_security_common_api.cc', - 'src/core/tsi/fake_transport_security.cc', - 'src/core/tsi/local_transport_security.cc', - 'src/core/tsi/transport_security.cc', - 'src/core/tsi/transport_security_grpc.cc', - 'third_party/upb/upb/message/accessors.c', - 'third_party/upb/upb/mini_descriptor/build_enum.c', - 'third_party/upb/upb/mini_descriptor/decode.c', - 'third_party/upb/upb/mini_descriptor/internal/base92.c', - 'third_party/upb/upb/mini_descriptor/internal/encode.c', - 'third_party/upb/upb/mini_descriptor/link.c', - 'third_party/upb/upb/wire/decode.c', - 'third_party/upb/upb/wire/decode_fast.c', - 'third_party/upb/upb/wire/encode.c', - 'third_party/upb/upb/wire/eps_copy_input_stream.c', - 'third_party/upb/upb/wire/reader.c', - ], - }, - { - 'target_name': 'gtest', - 'type': 'static_library', - 'dependencies': [ - 're2', - 'absl/container:flat_hash_set', - 'absl/debugging:failure_signal_handler', - 'absl/debugging:stacktrace', - 'absl/debugging:symbolize', - 'absl/flags:flag', - 'absl/flags:parse', - 'absl/flags:reflection', - 'absl/flags:usage', - 'absl/strings:strings', - 'absl/types:any', - 'absl/types:optional', - 'absl/types:variant', - ], - 'sources': [ - 'third_party/googletest/googlemock/src/gmock-cardinalities.cc', - 'third_party/googletest/googlemock/src/gmock-internal-utils.cc', - 'third_party/googletest/googlemock/src/gmock-matchers.cc', - 'third_party/googletest/googlemock/src/gmock-spec-builders.cc', - 'third_party/googletest/googlemock/src/gmock.cc', - 'third_party/googletest/googletest/src/gtest-assertion-result.cc', - 'third_party/googletest/googletest/src/gtest-death-test.cc', - 'third_party/googletest/googletest/src/gtest-filepath.cc', - 'third_party/googletest/googletest/src/gtest-matchers.cc', - 'third_party/googletest/googletest/src/gtest-port.cc', - 'third_party/googletest/googletest/src/gtest-printers.cc', - 'third_party/googletest/googletest/src/gtest-test-part.cc', - 'third_party/googletest/googletest/src/gtest-typed-test.cc', - 'third_party/googletest/googletest/src/gtest.cc', - ], - }, - { - 'target_name': 're2', - 'type': 'static_library', - 'dependencies': [ - ], - 'sources': [ - 'third_party/re2/re2/bitstate.cc', - 'third_party/re2/re2/compile.cc', - 'third_party/re2/re2/dfa.cc', - 'third_party/re2/re2/filtered_re2.cc', - 'third_party/re2/re2/mimics_pcre.cc', - 'third_party/re2/re2/nfa.cc', - 'third_party/re2/re2/onepass.cc', - 'third_party/re2/re2/parse.cc', - 'third_party/re2/re2/perl_groups.cc', - 'third_party/re2/re2/prefilter.cc', - 'third_party/re2/re2/prefilter_tree.cc', - 'third_party/re2/re2/prog.cc', - 'third_party/re2/re2/re2.cc', - 'third_party/re2/re2/regexp.cc', - 'third_party/re2/re2/set.cc', - 'third_party/re2/re2/simplify.cc', - 'third_party/re2/re2/stringpiece.cc', - 'third_party/re2/re2/tostring.cc', - 'third_party/re2/re2/unicode_casefold.cc', - 'third_party/re2/re2/unicode_groups.cc', - 'third_party/re2/util/rune.cc', - 'third_party/re2/util/strutil.cc', - ], - }, - { - 'target_name': 'upb_base_lib', - 'type': 'static_library', - 'dependencies': [ - ], - 'sources': [ - 'third_party/upb/upb/base/status.c', - ], - }, - { - 'target_name': 'upb_json_lib', - 'type': 'static_library', - 'dependencies': [ - 'upb_message_lib', - 'utf8_range_lib', - ], - 'sources': [ - 'src/core/ext/upb-gen/google/protobuf/descriptor.upb_minitable.c', - 'third_party/upb/upb/json/decode.c', - 'third_party/upb/upb/json/encode.c', - 'third_party/upb/upb/lex/atoi.c', - 'third_party/upb/upb/lex/round_trip.c', - 'third_party/upb/upb/lex/strtod.c', - 'third_party/upb/upb/lex/unicode.c', - 'third_party/upb/upb/message/accessors.c', - 'third_party/upb/upb/mini_descriptor/build_enum.c', - 'third_party/upb/upb/mini_descriptor/decode.c', - 'third_party/upb/upb/mini_descriptor/internal/base92.c', - 'third_party/upb/upb/mini_descriptor/internal/encode.c', - 'third_party/upb/upb/mini_descriptor/link.c', - 'third_party/upb/upb/reflection/def_pool.c', - 'third_party/upb/upb/reflection/def_type.c', - 'third_party/upb/upb/reflection/desc_state.c', - 'third_party/upb/upb/reflection/enum_def.c', - 'third_party/upb/upb/reflection/enum_reserved_range.c', - 'third_party/upb/upb/reflection/enum_value_def.c', - 'third_party/upb/upb/reflection/extension_range.c', - 'third_party/upb/upb/reflection/field_def.c', - 'third_party/upb/upb/reflection/file_def.c', - 'third_party/upb/upb/reflection/internal/def_builder.c', - 'third_party/upb/upb/reflection/internal/strdup2.c', - 'third_party/upb/upb/reflection/message.c', - 'third_party/upb/upb/reflection/message_def.c', - 'third_party/upb/upb/reflection/message_reserved_range.c', - 'third_party/upb/upb/reflection/method_def.c', - 'third_party/upb/upb/reflection/oneof_def.c', - 'third_party/upb/upb/reflection/service_def.c', - 'third_party/upb/upb/wire/decode.c', - 'third_party/upb/upb/wire/decode_fast.c', - 'third_party/upb/upb/wire/encode.c', - 'third_party/upb/upb/wire/eps_copy_input_stream.c', - 'third_party/upb/upb/wire/reader.c', - ], - }, - { - 'target_name': 'upb_mem_lib', - 'type': 'static_library', - 'dependencies': [ - ], - 'sources': [ - 'third_party/upb/upb/mem/alloc.c', - 'third_party/upb/upb/mem/arena.c', - ], - }, - { - 'target_name': 'upb_message_lib', - 'type': 'static_library', - 'dependencies': [ - 'upb_base_lib', - 'upb_mem_lib', - ], - 'sources': [ - 'third_party/upb/upb/hash/common.c', - 'third_party/upb/upb/message/array.c', - 'third_party/upb/upb/message/map.c', - 'third_party/upb/upb/message/map_sorter.c', - 'third_party/upb/upb/message/message.c', - 'third_party/upb/upb/mini_table/extension_registry.c', - 'third_party/upb/upb/mini_table/internal/message.c', - 'third_party/upb/upb/mini_table/message.c', - ], - }, - { - 'target_name': 'upb_textformat_lib', - 'type': 'static_library', - 'dependencies': [ - 'upb_message_lib', - 'utf8_range_lib', - ], - 'sources': [ - 'src/core/ext/upb-gen/google/protobuf/descriptor.upb_minitable.c', - 'third_party/upb/upb/lex/atoi.c', - 'third_party/upb/upb/lex/round_trip.c', - 'third_party/upb/upb/lex/strtod.c', - 'third_party/upb/upb/lex/unicode.c', - 'third_party/upb/upb/message/accessors.c', - 'third_party/upb/upb/mini_descriptor/build_enum.c', - 'third_party/upb/upb/mini_descriptor/decode.c', - 'third_party/upb/upb/mini_descriptor/internal/base92.c', - 'third_party/upb/upb/mini_descriptor/internal/encode.c', - 'third_party/upb/upb/mini_descriptor/link.c', - 'third_party/upb/upb/reflection/def_pool.c', - 'third_party/upb/upb/reflection/def_type.c', - 'third_party/upb/upb/reflection/desc_state.c', - 'third_party/upb/upb/reflection/enum_def.c', - 'third_party/upb/upb/reflection/enum_reserved_range.c', - 'third_party/upb/upb/reflection/enum_value_def.c', - 'third_party/upb/upb/reflection/extension_range.c', - 'third_party/upb/upb/reflection/field_def.c', - 'third_party/upb/upb/reflection/file_def.c', - 'third_party/upb/upb/reflection/internal/def_builder.c', - 'third_party/upb/upb/reflection/internal/strdup2.c', - 'third_party/upb/upb/reflection/message.c', - 'third_party/upb/upb/reflection/message_def.c', - 'third_party/upb/upb/reflection/message_reserved_range.c', - 'third_party/upb/upb/reflection/method_def.c', - 'third_party/upb/upb/reflection/oneof_def.c', - 'third_party/upb/upb/reflection/service_def.c', - 'third_party/upb/upb/text/encode.c', - 'third_party/upb/upb/wire/decode.c', - 'third_party/upb/upb/wire/decode_fast.c', - 'third_party/upb/upb/wire/encode.c', - 'third_party/upb/upb/wire/eps_copy_input_stream.c', - 'third_party/upb/upb/wire/reader.c', - ], - }, - { - 'target_name': 'utf8_range_lib', - 'type': 'static_library', - 'dependencies': [ - ], - 'sources': [ - 'third_party/utf8_range/naive.c', - 'third_party/utf8_range/range2-neon.c', - 'third_party/utf8_range/range2-sse.c', - ], - }, - { - 'target_name': 'z', - 'type': 'static_library', - 'dependencies': [ - ], - 'sources': [ - 'third_party/zlib/adler32.c', - 'third_party/zlib/compress.c', - 'third_party/zlib/crc32.c', - 'third_party/zlib/deflate.c', - 'third_party/zlib/infback.c', - 'third_party/zlib/inffast.c', - 'third_party/zlib/inflate.c', - 'third_party/zlib/inftrees.c', - 'third_party/zlib/trees.c', - 'third_party/zlib/uncompr.c', - 'third_party/zlib/zutil.c', - ], - }, - { - 'target_name': 'benchmark_helpers', - 'type': 'static_library', - 'dependencies': [ - 'benchmark', - 'grpc++_unsecure', - 'grpc_test_util_unsecure', - 'grpc++_test_config', - ], - 'sources': [ - 'src/proto/grpc/testing/echo.proto', - 'src/proto/grpc/testing/echo_messages.proto', - 'src/proto/grpc/testing/simple_messages.proto', - 'src/proto/grpc/testing/xds/v3/orca_load_report.proto', - 'test/core/util/cmdline.cc', - 'test/core/util/fuzzer_util.cc', - 'test/core/util/grpc_profiler.cc', - 'test/core/util/histogram.cc', - 'test/core/util/mock_endpoint.cc', - 'test/core/util/parse_hexstring.cc', - 'test/core/util/resolve_localhost_ip46.cc', - 'test/core/util/slice_splitter.cc', - 'test/core/util/tracer_util.cc', - 'test/cpp/microbenchmarks/helpers.cc', - ], - }, - { - 'target_name': 'grpc++', - 'type': 'static_library', - 'dependencies': [ - 'grpc', - 'protobuf', - ], - 'sources': [ - 'src/core/ext/transport/binder/client/binder_connector.cc', - 'src/core/ext/transport/binder/client/channel_create.cc', - 'src/core/ext/transport/binder/client/channel_create_impl.cc', - 'src/core/ext/transport/binder/client/connection_id_generator.cc', - 'src/core/ext/transport/binder/client/endpoint_binder_pool.cc', - 'src/core/ext/transport/binder/client/jni_utils.cc', - 'src/core/ext/transport/binder/client/security_policy_setting.cc', - 'src/core/ext/transport/binder/security_policy/binder_security_policy.cc', - 'src/core/ext/transport/binder/server/binder_server.cc', - 'src/core/ext/transport/binder/server/binder_server_credentials.cc', - 'src/core/ext/transport/binder/transport/binder_transport.cc', - 'src/core/ext/transport/binder/utils/ndk_binder.cc', - 'src/core/ext/transport/binder/utils/transport_stream_receiver_impl.cc', - 'src/core/ext/transport/binder/wire_format/binder_android.cc', - 'src/core/ext/transport/binder/wire_format/binder_constants.cc', - 'src/core/ext/transport/binder/wire_format/transaction.cc', - 'src/core/ext/transport/binder/wire_format/wire_reader_impl.cc', - 'src/core/ext/transport/binder/wire_format/wire_writer.cc', - 'src/cpp/client/channel_cc.cc', - 'src/cpp/client/client_callback.cc', - 'src/cpp/client/client_context.cc', - 'src/cpp/client/client_interceptor.cc', - 'src/cpp/client/client_stats_interceptor.cc', - 'src/cpp/client/create_channel.cc', - 'src/cpp/client/create_channel_internal.cc', - 'src/cpp/client/create_channel_posix.cc', - 'src/cpp/client/insecure_credentials.cc', - 'src/cpp/client/secure_credentials.cc', - 'src/cpp/client/xds_credentials.cc', - 'src/cpp/common/alarm.cc', - 'src/cpp/common/auth_property_iterator.cc', - 'src/cpp/common/channel_arguments.cc', - 'src/cpp/common/completion_queue_cc.cc', - 'src/cpp/common/resource_quota_cc.cc', - 'src/cpp/common/rpc_method.cc', - 'src/cpp/common/secure_auth_context.cc', - 'src/cpp/common/secure_channel_arguments.cc', - 'src/cpp/common/secure_create_auth_context.cc', - 'src/cpp/common/tls_certificate_provider.cc', - 'src/cpp/common/tls_certificate_verifier.cc', - 'src/cpp/common/tls_credentials_options.cc', - 'src/cpp/common/validate_service_config.cc', - 'src/cpp/common/version_cc.cc', - 'src/cpp/server/async_generic_service.cc', - 'src/cpp/server/backend_metric_recorder.cc', - 'src/cpp/server/channel_argument_option.cc', - 'src/cpp/server/create_default_thread_pool.cc', - 'src/cpp/server/external_connection_acceptor_impl.cc', - 'src/cpp/server/health/default_health_check_service.cc', - 'src/cpp/server/health/health_check_service.cc', - 'src/cpp/server/health/health_check_service_server_builder_option.cc', - 'src/cpp/server/insecure_server_credentials.cc', - 'src/cpp/server/secure_server_credentials.cc', - 'src/cpp/server/server_builder.cc', - 'src/cpp/server/server_callback.cc', - 'src/cpp/server/server_cc.cc', - 'src/cpp/server/server_context.cc', - 'src/cpp/server/server_posix.cc', - 'src/cpp/server/xds_server_builder.cc', - 'src/cpp/server/xds_server_credentials.cc', - 'src/cpp/thread_manager/thread_manager.cc', - 'src/cpp/util/byte_buffer_cc.cc', - 'src/cpp/util/status.cc', - 'src/cpp/util/string_ref.cc', - 'src/cpp/util/time_cc.cc', - ], - }, - { - 'target_name': 'grpc++_alts', - 'type': 'static_library', - 'dependencies': [ - 'grpc++', - ], - 'sources': [ - 'src/cpp/common/alts_context.cc', - 'src/cpp/common/alts_util.cc', - ], - }, - { - 'target_name': 'grpc++_error_details', - 'type': 'static_library', - 'dependencies': [ - 'grpc++', - ], - 'sources': [ - 'src/cpp/util/error_details.cc', - ], - }, - { - 'target_name': 'grpc++_reflection', - 'type': 'static_library', - 'dependencies': [ - 'grpc++', - ], - 'sources': [ - 'src/proto/grpc/reflection/v1/reflection.proto', - 'src/proto/grpc/reflection/v1alpha/reflection.proto', - 'src/cpp/ext/proto_server_reflection.cc', - 'src/cpp/ext/proto_server_reflection_plugin.cc', - ], - }, - { - 'target_name': 'grpc++_test', - 'type': 'static_library', - 'dependencies': [ - 'gtest', - 'grpc++', - ], - 'sources': [ - 'src/cpp/client/channel_test_peer.cc', - ], - }, - { - 'target_name': 'grpc++_test_config', - 'type': 'static_library', - 'dependencies': [ - 'absl/flags:parse', - 'gpr', - ], - 'sources': [ - 'test/cpp/util/test_config_cc.cc', - ], - }, - { - 'target_name': 'grpc++_test_util', - 'type': 'static_library', - 'dependencies': [ - 'grpc++', - 'grpc_test_util', - ], - 'sources': [ - 'src/core/lib/gpr/subprocess_posix.cc', - 'src/core/lib/gpr/subprocess_windows.cc', - 'test/core/end2end/data/client_certs.cc', - 'test/core/end2end/data/server1_cert.cc', - 'test/core/end2end/data/server1_key.cc', - 'test/core/end2end/data/test_root_cert.cc', - 'test/core/util/cmdline.cc', - 'test/core/util/fuzzer_util.cc', - 'test/core/util/grpc_profiler.cc', - 'test/core/util/histogram.cc', - 'test/core/util/mock_endpoint.cc', - 'test/core/util/parse_hexstring.cc', - 'test/core/util/resolve_localhost_ip46.cc', - 'test/core/util/slice_splitter.cc', - 'test/core/util/tracer_util.cc', - 'test/cpp/util/byte_buffer_proto_helper.cc', - 'test/cpp/util/create_test_channel.cc', - 'test/cpp/util/string_ref_helper.cc', - 'test/cpp/util/subprocess.cc', - 'test/cpp/util/test_credentials_provider.cc', - ], - }, - { - 'target_name': 'grpc++_unsecure', - 'type': 'static_library', - 'dependencies': [ - 'grpc_unsecure', - 'protobuf', - ], - 'sources': [ - 'src/cpp/client/channel_cc.cc', - 'src/cpp/client/client_callback.cc', - 'src/cpp/client/client_context.cc', - 'src/cpp/client/client_interceptor.cc', - 'src/cpp/client/client_stats_interceptor.cc', - 'src/cpp/client/create_channel.cc', - 'src/cpp/client/create_channel_internal.cc', - 'src/cpp/client/create_channel_posix.cc', - 'src/cpp/client/insecure_credentials.cc', - 'src/cpp/common/alarm.cc', - 'src/cpp/common/channel_arguments.cc', - 'src/cpp/common/completion_queue_cc.cc', - 'src/cpp/common/insecure_create_auth_context.cc', - 'src/cpp/common/resource_quota_cc.cc', - 'src/cpp/common/rpc_method.cc', - 'src/cpp/common/validate_service_config.cc', - 'src/cpp/common/version_cc.cc', - 'src/cpp/server/async_generic_service.cc', - 'src/cpp/server/backend_metric_recorder.cc', - 'src/cpp/server/channel_argument_option.cc', - 'src/cpp/server/create_default_thread_pool.cc', - 'src/cpp/server/external_connection_acceptor_impl.cc', - 'src/cpp/server/health/default_health_check_service.cc', - 'src/cpp/server/health/health_check_service.cc', - 'src/cpp/server/health/health_check_service_server_builder_option.cc', - 'src/cpp/server/insecure_server_credentials.cc', - 'src/cpp/server/server_builder.cc', - 'src/cpp/server/server_callback.cc', - 'src/cpp/server/server_cc.cc', - 'src/cpp/server/server_context.cc', - 'src/cpp/server/server_posix.cc', - 'src/cpp/thread_manager/thread_manager.cc', - 'src/cpp/util/byte_buffer_cc.cc', - 'src/cpp/util/status.cc', - 'src/cpp/util/string_ref.cc', - 'src/cpp/util/time_cc.cc', - ], - }, - { - 'target_name': 'grpc_authorization_provider', - 'type': 'static_library', - 'dependencies': [ - 'upb_message_lib', - 're2', - 'utf8_range_lib', - 'z', - 'absl/base:config', - 'absl/base:no_destructor', - 'absl/cleanup:cleanup', - 'absl/container:flat_hash_map', - 'absl/container:flat_hash_set', - 'absl/container:inlined_vector', - 'absl/functional:function_ref', - 'absl/hash:hash', - 'absl/meta:type_traits', - 'absl/random:bit_gen_ref', - 'absl/random:distributions', - 'absl/status:statusor', - 'absl/types:span', - 'absl/utility:utility', - 'cares', - 'gpr', - 'address_sorting', - ], - 'sources': [ - 'src/core/ext/upb-gen/google/protobuf/any.upb_minitable.c', - 'src/core/ext/upb-gen/google/protobuf/descriptor.upb_minitable.c', - 'src/core/ext/upb-gen/google/rpc/status.upb_minitable.c', - 'src/core/ext/upb-gen/src/proto/grpc/gcp/altscontext.upb_minitable.c', - 'src/core/ext/upb-gen/src/proto/grpc/gcp/handshaker.upb_minitable.c', - 'src/core/ext/upb-gen/src/proto/grpc/gcp/transport_security_common.upb_minitable.c', - 'src/core/lib/address_utils/parse_address.cc', - 'src/core/lib/address_utils/sockaddr_utils.cc', - 'src/core/lib/backoff/backoff.cc', - 'src/core/lib/backoff/random_early_detection.cc', - 'src/core/lib/channel/call_tracer.cc', - 'src/core/lib/channel/channel_args.cc', - 'src/core/lib/channel/channel_args_preconditioning.cc', - 'src/core/lib/channel/channel_stack.cc', - 'src/core/lib/channel/channel_stack_builder.cc', - 'src/core/lib/channel/channel_stack_builder_impl.cc', - 'src/core/lib/channel/channel_stack_trace.cc', - 'src/core/lib/channel/channel_trace.cc', - 'src/core/lib/channel/channelz.cc', - 'src/core/lib/channel/channelz_registry.cc', - 'src/core/lib/channel/connected_channel.cc', - 'src/core/lib/channel/metrics.cc', - 'src/core/lib/channel/promise_based_filter.cc', - 'src/core/lib/channel/status_util.cc', - 'src/core/lib/compression/compression.cc', - 'src/core/lib/compression/compression_internal.cc', - 'src/core/lib/compression/message_compress.cc', - 'src/core/lib/config/core_configuration.cc', - 'src/core/lib/debug/event_log.cc', - 'src/core/lib/debug/histogram_view.cc', - 'src/core/lib/debug/stats.cc', - 'src/core/lib/debug/stats_data.cc', - 'src/core/lib/debug/trace.cc', - 'src/core/lib/event_engine/ares_resolver.cc', - 'src/core/lib/event_engine/cf_engine/cf_engine.cc', - 'src/core/lib/event_engine/cf_engine/cfstream_endpoint.cc', - 'src/core/lib/event_engine/cf_engine/dns_service_resolver.cc', - 'src/core/lib/event_engine/channel_args_endpoint_config.cc', - 'src/core/lib/event_engine/default_event_engine.cc', - 'src/core/lib/event_engine/default_event_engine_factory.cc', - 'src/core/lib/event_engine/event_engine.cc', - 'src/core/lib/event_engine/forkable.cc', - 'src/core/lib/event_engine/posix_engine/ev_epoll1_linux.cc', - 'src/core/lib/event_engine/posix_engine/ev_poll_posix.cc', - 'src/core/lib/event_engine/posix_engine/event_poller_posix_default.cc', - 'src/core/lib/event_engine/posix_engine/internal_errqueue.cc', - 'src/core/lib/event_engine/posix_engine/lockfree_event.cc', - 'src/core/lib/event_engine/posix_engine/native_posix_dns_resolver.cc', - 'src/core/lib/event_engine/posix_engine/posix_endpoint.cc', - 'src/core/lib/event_engine/posix_engine/posix_engine.cc', - 'src/core/lib/event_engine/posix_engine/posix_engine_listener.cc', - 'src/core/lib/event_engine/posix_engine/posix_engine_listener_utils.cc', - 'src/core/lib/event_engine/posix_engine/tcp_socket_utils.cc', - 'src/core/lib/event_engine/posix_engine/timer.cc', - 'src/core/lib/event_engine/posix_engine/timer_heap.cc', - 'src/core/lib/event_engine/posix_engine/timer_manager.cc', - 'src/core/lib/event_engine/posix_engine/traced_buffer_list.cc', - 'src/core/lib/event_engine/posix_engine/wakeup_fd_eventfd.cc', - 'src/core/lib/event_engine/posix_engine/wakeup_fd_pipe.cc', - 'src/core/lib/event_engine/posix_engine/wakeup_fd_posix_default.cc', - 'src/core/lib/event_engine/resolved_address.cc', - 'src/core/lib/event_engine/shim.cc', - 'src/core/lib/event_engine/slice.cc', - 'src/core/lib/event_engine/slice_buffer.cc', - 'src/core/lib/event_engine/tcp_socket_utils.cc', - 'src/core/lib/event_engine/thread_pool/thread_count.cc', - 'src/core/lib/event_engine/thread_pool/thread_pool_factory.cc', - 'src/core/lib/event_engine/thread_pool/work_stealing_thread_pool.cc', - 'src/core/lib/event_engine/thready_event_engine/thready_event_engine.cc', - 'src/core/lib/event_engine/time_util.cc', - 'src/core/lib/event_engine/trace.cc', - 'src/core/lib/event_engine/utils.cc', - 'src/core/lib/event_engine/windows/grpc_polled_fd_windows.cc', - 'src/core/lib/event_engine/windows/iocp.cc', - 'src/core/lib/event_engine/windows/native_windows_dns_resolver.cc', - 'src/core/lib/event_engine/windows/win_socket.cc', - 'src/core/lib/event_engine/windows/windows_endpoint.cc', - 'src/core/lib/event_engine/windows/windows_engine.cc', - 'src/core/lib/event_engine/windows/windows_listener.cc', - 'src/core/lib/event_engine/work_queue/basic_work_queue.cc', - 'src/core/lib/experiments/config.cc', - 'src/core/lib/experiments/experiments.cc', - 'src/core/lib/gprpp/load_file.cc', - 'src/core/lib/gprpp/per_cpu.cc', - 'src/core/lib/gprpp/ref_counted_string.cc', - 'src/core/lib/gprpp/status_helper.cc', - 'src/core/lib/gprpp/time.cc', - 'src/core/lib/gprpp/time_averaged_stats.cc', - 'src/core/lib/gprpp/validation_errors.cc', - 'src/core/lib/gprpp/work_serializer.cc', - 'src/core/lib/handshaker/proxy_mapper_registry.cc', - 'src/core/lib/iomgr/buffer_list.cc', - 'src/core/lib/iomgr/call_combiner.cc', - 'src/core/lib/iomgr/cfstream_handle.cc', - 'src/core/lib/iomgr/closure.cc', - 'src/core/lib/iomgr/combiner.cc', - 'src/core/lib/iomgr/dualstack_socket_posix.cc', - 'src/core/lib/iomgr/endpoint.cc', - 'src/core/lib/iomgr/endpoint_cfstream.cc', - 'src/core/lib/iomgr/endpoint_pair_posix.cc', - 'src/core/lib/iomgr/endpoint_pair_windows.cc', - 'src/core/lib/iomgr/error.cc', - 'src/core/lib/iomgr/error_cfstream.cc', - 'src/core/lib/iomgr/ev_apple.cc', - 'src/core/lib/iomgr/ev_epoll1_linux.cc', - 'src/core/lib/iomgr/ev_poll_posix.cc', - 'src/core/lib/iomgr/ev_posix.cc', - 'src/core/lib/iomgr/ev_windows.cc', - 'src/core/lib/iomgr/event_engine_shims/closure.cc', - 'src/core/lib/iomgr/event_engine_shims/endpoint.cc', - 'src/core/lib/iomgr/event_engine_shims/tcp_client.cc', - 'src/core/lib/iomgr/exec_ctx.cc', - 'src/core/lib/iomgr/executor.cc', - 'src/core/lib/iomgr/fork_posix.cc', - 'src/core/lib/iomgr/fork_windows.cc', - 'src/core/lib/iomgr/gethostname_fallback.cc', - 'src/core/lib/iomgr/gethostname_host_name_max.cc', - 'src/core/lib/iomgr/gethostname_sysconf.cc', - 'src/core/lib/iomgr/grpc_if_nametoindex_posix.cc', - 'src/core/lib/iomgr/grpc_if_nametoindex_unsupported.cc', - 'src/core/lib/iomgr/internal_errqueue.cc', - 'src/core/lib/iomgr/iocp_windows.cc', - 'src/core/lib/iomgr/iomgr.cc', - 'src/core/lib/iomgr/iomgr_internal.cc', - 'src/core/lib/iomgr/iomgr_posix.cc', - 'src/core/lib/iomgr/iomgr_posix_cfstream.cc', - 'src/core/lib/iomgr/iomgr_windows.cc', - 'src/core/lib/iomgr/lockfree_event.cc', - 'src/core/lib/iomgr/polling_entity.cc', - 'src/core/lib/iomgr/pollset.cc', - 'src/core/lib/iomgr/pollset_set.cc', - 'src/core/lib/iomgr/pollset_set_windows.cc', - 'src/core/lib/iomgr/pollset_windows.cc', - 'src/core/lib/iomgr/resolve_address.cc', - 'src/core/lib/iomgr/resolve_address_posix.cc', - 'src/core/lib/iomgr/resolve_address_windows.cc', - 'src/core/lib/iomgr/sockaddr_utils_posix.cc', - 'src/core/lib/iomgr/socket_factory_posix.cc', - 'src/core/lib/iomgr/socket_mutator.cc', - 'src/core/lib/iomgr/socket_utils_common_posix.cc', - 'src/core/lib/iomgr/socket_utils_linux.cc', - 'src/core/lib/iomgr/socket_utils_posix.cc', - 'src/core/lib/iomgr/socket_utils_windows.cc', - 'src/core/lib/iomgr/socket_windows.cc', - 'src/core/lib/iomgr/systemd_utils.cc', - 'src/core/lib/iomgr/tcp_client.cc', - 'src/core/lib/iomgr/tcp_client_cfstream.cc', - 'src/core/lib/iomgr/tcp_client_posix.cc', - 'src/core/lib/iomgr/tcp_client_windows.cc', - 'src/core/lib/iomgr/tcp_posix.cc', - 'src/core/lib/iomgr/tcp_server.cc', - 'src/core/lib/iomgr/tcp_server_posix.cc', - 'src/core/lib/iomgr/tcp_server_utils_posix_common.cc', - 'src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.cc', - 'src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.cc', - 'src/core/lib/iomgr/tcp_server_windows.cc', - 'src/core/lib/iomgr/tcp_windows.cc', - 'src/core/lib/iomgr/timer.cc', - 'src/core/lib/iomgr/timer_generic.cc', - 'src/core/lib/iomgr/timer_heap.cc', - 'src/core/lib/iomgr/timer_manager.cc', - 'src/core/lib/iomgr/unix_sockets_posix.cc', - 'src/core/lib/iomgr/unix_sockets_posix_noop.cc', - 'src/core/lib/iomgr/vsock.cc', - 'src/core/lib/iomgr/wakeup_fd_eventfd.cc', - 'src/core/lib/iomgr/wakeup_fd_nospecial.cc', - 'src/core/lib/iomgr/wakeup_fd_pipe.cc', - 'src/core/lib/iomgr/wakeup_fd_posix.cc', - 'src/core/lib/json/json_reader.cc', - 'src/core/lib/json/json_writer.cc', - 'src/core/lib/matchers/matchers.cc', - 'src/core/lib/promise/activity.cc', - 'src/core/lib/promise/party.cc', - 'src/core/lib/promise/trace.cc', - 'src/core/lib/resource_quota/api.cc', - 'src/core/lib/resource_quota/arena.cc', - 'src/core/lib/resource_quota/connection_quota.cc', - 'src/core/lib/resource_quota/memory_quota.cc', - 'src/core/lib/resource_quota/periodic_update.cc', - 'src/core/lib/resource_quota/resource_quota.cc', - 'src/core/lib/resource_quota/thread_quota.cc', - 'src/core/lib/resource_quota/trace.cc', - 'src/core/lib/security/authorization/audit_logging.cc', - 'src/core/lib/security/authorization/authorization_policy_provider_vtable.cc', - 'src/core/lib/security/authorization/evaluate_args.cc', - 'src/core/lib/security/authorization/grpc_authorization_engine.cc', - 'src/core/lib/security/authorization/grpc_authorization_policy_provider.cc', - 'src/core/lib/security/authorization/grpc_server_authz_filter.cc', - 'src/core/lib/security/authorization/matchers.cc', - 'src/core/lib/security/authorization/rbac_policy.cc', - 'src/core/lib/security/authorization/rbac_translator.cc', - 'src/core/lib/security/authorization/stdout_logger.cc', - 'src/core/lib/security/certificate_provider/certificate_provider_registry.cc', - 'src/core/lib/security/context/security_context.cc', - 'src/core/lib/security/credentials/alts/check_gcp_environment.cc', - 'src/core/lib/security/credentials/alts/check_gcp_environment_linux.cc', - 'src/core/lib/security/credentials/alts/check_gcp_environment_no_op.cc', - 'src/core/lib/security/credentials/alts/check_gcp_environment_windows.cc', - 'src/core/lib/security/credentials/alts/grpc_alts_credentials_client_options.cc', - 'src/core/lib/security/credentials/alts/grpc_alts_credentials_options.cc', - 'src/core/lib/security/credentials/alts/grpc_alts_credentials_server_options.cc', - 'src/core/lib/security/credentials/call_creds_util.cc', - 'src/core/lib/security/credentials/composite/composite_credentials.cc', - 'src/core/lib/security/credentials/credentials.cc', - 'src/core/lib/security/credentials/plugin/plugin_credentials.cc', - 'src/core/lib/security/credentials/tls/tls_utils.cc', - 'src/core/lib/security/security_connector/load_system_roots_fallback.cc', - 'src/core/lib/security/security_connector/load_system_roots_supported.cc', - 'src/core/lib/security/security_connector/load_system_roots_windows.cc', - 'src/core/lib/security/security_connector/security_connector.cc', - 'src/core/lib/security/transport/client_auth_filter.cc', - 'src/core/lib/security/transport/legacy_server_auth_filter.cc', - 'src/core/lib/security/transport/secure_endpoint.cc', - 'src/core/lib/security/transport/security_handshaker.cc', - 'src/core/lib/security/transport/server_auth_filter.cc', - 'src/core/lib/security/transport/tsi_error.cc', - 'src/core/lib/security/util/json_util.cc', - 'src/core/lib/slice/b64.cc', - 'src/core/lib/slice/percent_encoding.cc', - 'src/core/lib/slice/slice.cc', - 'src/core/lib/slice/slice_buffer.cc', - 'src/core/lib/slice/slice_refcount.cc', - 'src/core/lib/slice/slice_string_helpers.cc', - 'src/core/lib/surface/api_trace.cc', - 'src/core/lib/surface/byte_buffer.cc', - 'src/core/lib/surface/byte_buffer_reader.cc', - 'src/core/lib/surface/call.cc', - 'src/core/lib/surface/call_details.cc', - 'src/core/lib/surface/call_log_batch.cc', - 'src/core/lib/surface/channel.cc', - 'src/core/lib/surface/channel_init.cc', - 'src/core/lib/surface/channel_stack_type.cc', - 'src/core/lib/surface/completion_queue.cc', - 'src/core/lib/surface/completion_queue_factory.cc', - 'src/core/lib/surface/event_string.cc', - 'src/core/lib/surface/init_internally.cc', - 'src/core/lib/surface/lame_client.cc', - 'src/core/lib/surface/metadata_array.cc', - 'src/core/lib/surface/validate_metadata.cc', - 'src/core/lib/surface/version.cc', - 'src/core/lib/surface/wait_for_cq_end_op.cc', - 'src/core/lib/transport/batch_builder.cc', - 'src/core/lib/transport/call_factory.cc', - 'src/core/lib/transport/call_filters.cc', - 'src/core/lib/transport/call_final_info.cc', - 'src/core/lib/transport/call_size_estimator.cc', - 'src/core/lib/transport/call_spine.cc', - 'src/core/lib/transport/connectivity_state.cc', - 'src/core/lib/transport/error_utils.cc', - 'src/core/lib/transport/handshaker.cc', - 'src/core/lib/transport/handshaker_registry.cc', - 'src/core/lib/transport/message.cc', - 'src/core/lib/transport/metadata.cc', - 'src/core/lib/transport/metadata_batch.cc', - 'src/core/lib/transport/parsed_metadata.cc', - 'src/core/lib/transport/status_conversion.cc', - 'src/core/lib/transport/timeout_encoding.cc', - 'src/core/lib/transport/transport.cc', - 'src/core/lib/transport/transport_op_string.cc', - 'src/core/lib/uri/uri_parser.cc', - 'src/core/load_balancing/lb_policy.cc', - 'src/core/load_balancing/lb_policy_registry.cc', - 'src/core/resolver/endpoint_addresses.cc', - 'src/core/resolver/resolver.cc', - 'src/core/resolver/resolver_registry.cc', - 'src/core/service_config/service_config_parser.cc', - 'src/core/tsi/alts/handshaker/transport_security_common_api.cc', - 'src/core/tsi/transport_security.cc', - 'src/core/tsi/transport_security_grpc.cc', - 'third_party/upb/upb/message/accessors.c', - 'third_party/upb/upb/mini_descriptor/build_enum.c', - 'third_party/upb/upb/mini_descriptor/decode.c', - 'third_party/upb/upb/mini_descriptor/internal/base92.c', - 'third_party/upb/upb/mini_descriptor/internal/encode.c', - 'third_party/upb/upb/mini_descriptor/link.c', - 'third_party/upb/upb/wire/decode.c', - 'third_party/upb/upb/wire/decode_fast.c', - 'third_party/upb/upb/wire/encode.c', - 'third_party/upb/upb/wire/eps_copy_input_stream.c', - 'third_party/upb/upb/wire/reader.c', - ], - }, - { - 'target_name': 'grpc_plugin_support', - 'type': 'static_library', - 'dependencies': [ - 'protobuf', - 'protoc', - ], - 'sources': [ - 'src/compiler/cpp_generator.cc', - 'src/compiler/csharp_generator.cc', - 'src/compiler/node_generator.cc', - 'src/compiler/objective_c_generator.cc', - 'src/compiler/php_generator.cc', - 'src/compiler/proto_parser_helper.cc', - 'src/compiler/python_generator.cc', - 'src/compiler/ruby_generator.cc', - ], - }, - { - 'target_name': 'grpcpp_channelz', - 'type': 'static_library', - 'dependencies': [ - 'grpc++', - ], - 'sources': [ - 'src/proto/grpc/channelz/channelz.proto', - 'src/cpp/server/channelz/channelz_service.cc', - 'src/cpp/server/channelz/channelz_service_plugin.cc', - ], - }, - { - 'target_name': 'grpcpp_otel_plugin', - 'type': 'static_library', - 'dependencies': [ - 'grpc++', - 'opentelemetry-cpp::api', - ], - 'sources': [ - 'src/cpp/ext/otel/otel_client_filter.cc', - 'src/cpp/ext/otel/otel_plugin.cc', - 'src/cpp/ext/otel/otel_server_call_tracer.cc', - ], - }, - { - 'target_name': 'boringssl', - 'type': 'static_library', - 'dependencies': [ - ], - 'sources': [ - 'third_party/boringssl-with-bazel/err_data.c', - 'third_party/boringssl-with-bazel/src/crypto/asn1/a_bitstr.c', - 'third_party/boringssl-with-bazel/src/crypto/asn1/a_bool.c', - 'third_party/boringssl-with-bazel/src/crypto/asn1/a_d2i_fp.c', - 'third_party/boringssl-with-bazel/src/crypto/asn1/a_dup.c', - 'third_party/boringssl-with-bazel/src/crypto/asn1/a_gentm.c', - 'third_party/boringssl-with-bazel/src/crypto/asn1/a_i2d_fp.c', - 'third_party/boringssl-with-bazel/src/crypto/asn1/a_int.c', - 'third_party/boringssl-with-bazel/src/crypto/asn1/a_mbstr.c', - 'third_party/boringssl-with-bazel/src/crypto/asn1/a_object.c', - 'third_party/boringssl-with-bazel/src/crypto/asn1/a_octet.c', - 'third_party/boringssl-with-bazel/src/crypto/asn1/a_strex.c', - 'third_party/boringssl-with-bazel/src/crypto/asn1/a_strnid.c', - 'third_party/boringssl-with-bazel/src/crypto/asn1/a_time.c', - 'third_party/boringssl-with-bazel/src/crypto/asn1/a_type.c', - 'third_party/boringssl-with-bazel/src/crypto/asn1/a_utctm.c', - 'third_party/boringssl-with-bazel/src/crypto/asn1/asn1_lib.c', - 'third_party/boringssl-with-bazel/src/crypto/asn1/asn1_par.c', - 'third_party/boringssl-with-bazel/src/crypto/asn1/asn_pack.c', - 'third_party/boringssl-with-bazel/src/crypto/asn1/f_int.c', - 'third_party/boringssl-with-bazel/src/crypto/asn1/f_string.c', - 'third_party/boringssl-with-bazel/src/crypto/asn1/posix_time.c', - 'third_party/boringssl-with-bazel/src/crypto/asn1/tasn_dec.c', - 'third_party/boringssl-with-bazel/src/crypto/asn1/tasn_enc.c', - 'third_party/boringssl-with-bazel/src/crypto/asn1/tasn_fre.c', - 'third_party/boringssl-with-bazel/src/crypto/asn1/tasn_new.c', - 'third_party/boringssl-with-bazel/src/crypto/asn1/tasn_typ.c', - 'third_party/boringssl-with-bazel/src/crypto/asn1/tasn_utl.c', - 'third_party/boringssl-with-bazel/src/crypto/base64/base64.c', - 'third_party/boringssl-with-bazel/src/crypto/bio/bio.c', - 'third_party/boringssl-with-bazel/src/crypto/bio/bio_mem.c', - 'third_party/boringssl-with-bazel/src/crypto/bio/connect.c', - 'third_party/boringssl-with-bazel/src/crypto/bio/errno.c', - 'third_party/boringssl-with-bazel/src/crypto/bio/fd.c', - 'third_party/boringssl-with-bazel/src/crypto/bio/file.c', - 'third_party/boringssl-with-bazel/src/crypto/bio/hexdump.c', - 'third_party/boringssl-with-bazel/src/crypto/bio/pair.c', - 'third_party/boringssl-with-bazel/src/crypto/bio/printf.c', - 'third_party/boringssl-with-bazel/src/crypto/bio/socket.c', - 'third_party/boringssl-with-bazel/src/crypto/bio/socket_helper.c', - 'third_party/boringssl-with-bazel/src/crypto/blake2/blake2.c', - 'third_party/boringssl-with-bazel/src/crypto/bn_extra/bn_asn1.c', - 'third_party/boringssl-with-bazel/src/crypto/bn_extra/convert.c', - 'third_party/boringssl-with-bazel/src/crypto/buf/buf.c', - 'third_party/boringssl-with-bazel/src/crypto/bytestring/asn1_compat.c', - 'third_party/boringssl-with-bazel/src/crypto/bytestring/ber.c', - 'third_party/boringssl-with-bazel/src/crypto/bytestring/cbb.c', - 'third_party/boringssl-with-bazel/src/crypto/bytestring/cbs.c', - 'third_party/boringssl-with-bazel/src/crypto/bytestring/unicode.c', - 'third_party/boringssl-with-bazel/src/crypto/chacha/chacha.c', - 'third_party/boringssl-with-bazel/src/crypto/cipher_extra/cipher_extra.c', - 'third_party/boringssl-with-bazel/src/crypto/cipher_extra/derive_key.c', - 'third_party/boringssl-with-bazel/src/crypto/cipher_extra/e_aesctrhmac.c', - 'third_party/boringssl-with-bazel/src/crypto/cipher_extra/e_aesgcmsiv.c', - 'third_party/boringssl-with-bazel/src/crypto/cipher_extra/e_chacha20poly1305.c', - 'third_party/boringssl-with-bazel/src/crypto/cipher_extra/e_des.c', - 'third_party/boringssl-with-bazel/src/crypto/cipher_extra/e_null.c', - 'third_party/boringssl-with-bazel/src/crypto/cipher_extra/e_rc2.c', - 'third_party/boringssl-with-bazel/src/crypto/cipher_extra/e_rc4.c', - 'third_party/boringssl-with-bazel/src/crypto/cipher_extra/e_tls.c', - 'third_party/boringssl-with-bazel/src/crypto/cipher_extra/tls_cbc.c', - 'third_party/boringssl-with-bazel/src/crypto/conf/conf.c', - 'third_party/boringssl-with-bazel/src/crypto/cpu_aarch64_apple.c', - 'third_party/boringssl-with-bazel/src/crypto/cpu_aarch64_fuchsia.c', - 'third_party/boringssl-with-bazel/src/crypto/cpu_aarch64_linux.c', - 'third_party/boringssl-with-bazel/src/crypto/cpu_aarch64_openbsd.c', - 'third_party/boringssl-with-bazel/src/crypto/cpu_aarch64_sysreg.c', - 'third_party/boringssl-with-bazel/src/crypto/cpu_aarch64_win.c', - 'third_party/boringssl-with-bazel/src/crypto/cpu_arm_freebsd.c', - 'third_party/boringssl-with-bazel/src/crypto/cpu_arm_linux.c', - 'third_party/boringssl-with-bazel/src/crypto/cpu_intel.c', - 'third_party/boringssl-with-bazel/src/crypto/crypto.c', - 'third_party/boringssl-with-bazel/src/crypto/curve25519/curve25519.c', - 'third_party/boringssl-with-bazel/src/crypto/curve25519/curve25519_64_adx.c', - 'third_party/boringssl-with-bazel/src/crypto/curve25519/spake25519.c', - 'third_party/boringssl-with-bazel/src/crypto/des/des.c', - 'third_party/boringssl-with-bazel/src/crypto/dh_extra/dh_asn1.c', - 'third_party/boringssl-with-bazel/src/crypto/dh_extra/params.c', - 'third_party/boringssl-with-bazel/src/crypto/digest_extra/digest_extra.c', - 'third_party/boringssl-with-bazel/src/crypto/dsa/dsa.c', - 'third_party/boringssl-with-bazel/src/crypto/dsa/dsa_asn1.c', - 'third_party/boringssl-with-bazel/src/crypto/ec_extra/ec_asn1.c', - 'third_party/boringssl-with-bazel/src/crypto/ec_extra/ec_derive.c', - 'third_party/boringssl-with-bazel/src/crypto/ec_extra/hash_to_curve.c', - 'third_party/boringssl-with-bazel/src/crypto/ecdh_extra/ecdh_extra.c', - 'third_party/boringssl-with-bazel/src/crypto/ecdsa_extra/ecdsa_asn1.c', - 'third_party/boringssl-with-bazel/src/crypto/engine/engine.c', - 'third_party/boringssl-with-bazel/src/crypto/err/err.c', - 'third_party/boringssl-with-bazel/src/crypto/evp/evp.c', - 'third_party/boringssl-with-bazel/src/crypto/evp/evp_asn1.c', - 'third_party/boringssl-with-bazel/src/crypto/evp/evp_ctx.c', - 'third_party/boringssl-with-bazel/src/crypto/evp/p_dsa_asn1.c', - 'third_party/boringssl-with-bazel/src/crypto/evp/p_ec.c', - 'third_party/boringssl-with-bazel/src/crypto/evp/p_ec_asn1.c', - 'third_party/boringssl-with-bazel/src/crypto/evp/p_ed25519.c', - 'third_party/boringssl-with-bazel/src/crypto/evp/p_ed25519_asn1.c', - 'third_party/boringssl-with-bazel/src/crypto/evp/p_hkdf.c', - 'third_party/boringssl-with-bazel/src/crypto/evp/p_rsa.c', - 'third_party/boringssl-with-bazel/src/crypto/evp/p_rsa_asn1.c', - 'third_party/boringssl-with-bazel/src/crypto/evp/p_x25519.c', - 'third_party/boringssl-with-bazel/src/crypto/evp/p_x25519_asn1.c', - 'third_party/boringssl-with-bazel/src/crypto/evp/pbkdf.c', - 'third_party/boringssl-with-bazel/src/crypto/evp/print.c', - 'third_party/boringssl-with-bazel/src/crypto/evp/scrypt.c', - 'third_party/boringssl-with-bazel/src/crypto/evp/sign.c', - 'third_party/boringssl-with-bazel/src/crypto/ex_data.c', - 'third_party/boringssl-with-bazel/src/crypto/fipsmodule/bcm.c', - 'third_party/boringssl-with-bazel/src/crypto/fipsmodule/fips_shared_support.c', - 'third_party/boringssl-with-bazel/src/crypto/hpke/hpke.c', - 'third_party/boringssl-with-bazel/src/crypto/hrss/hrss.c', - 'third_party/boringssl-with-bazel/src/crypto/keccak/keccak.c', - 'third_party/boringssl-with-bazel/src/crypto/kyber/kyber.c', - 'third_party/boringssl-with-bazel/src/crypto/lhash/lhash.c', - 'third_party/boringssl-with-bazel/src/crypto/mem.c', - 'third_party/boringssl-with-bazel/src/crypto/obj/obj.c', - 'third_party/boringssl-with-bazel/src/crypto/obj/obj_xref.c', - 'third_party/boringssl-with-bazel/src/crypto/pem/pem_all.c', - 'third_party/boringssl-with-bazel/src/crypto/pem/pem_info.c', - 'third_party/boringssl-with-bazel/src/crypto/pem/pem_lib.c', - 'third_party/boringssl-with-bazel/src/crypto/pem/pem_oth.c', - 'third_party/boringssl-with-bazel/src/crypto/pem/pem_pk8.c', - 'third_party/boringssl-with-bazel/src/crypto/pem/pem_pkey.c', - 'third_party/boringssl-with-bazel/src/crypto/pem/pem_x509.c', - 'third_party/boringssl-with-bazel/src/crypto/pem/pem_xaux.c', - 'third_party/boringssl-with-bazel/src/crypto/pkcs7/pkcs7.c', - 'third_party/boringssl-with-bazel/src/crypto/pkcs7/pkcs7_x509.c', - 'third_party/boringssl-with-bazel/src/crypto/pkcs8/p5_pbev2.c', - 'third_party/boringssl-with-bazel/src/crypto/pkcs8/pkcs8.c', - 'third_party/boringssl-with-bazel/src/crypto/pkcs8/pkcs8_x509.c', - 'third_party/boringssl-with-bazel/src/crypto/poly1305/poly1305.c', - 'third_party/boringssl-with-bazel/src/crypto/poly1305/poly1305_arm.c', - 'third_party/boringssl-with-bazel/src/crypto/poly1305/poly1305_vec.c', - 'third_party/boringssl-with-bazel/src/crypto/pool/pool.c', - 'third_party/boringssl-with-bazel/src/crypto/rand_extra/deterministic.c', - 'third_party/boringssl-with-bazel/src/crypto/rand_extra/forkunsafe.c', - 'third_party/boringssl-with-bazel/src/crypto/rand_extra/getentropy.c', - 'third_party/boringssl-with-bazel/src/crypto/rand_extra/ios.c', - 'third_party/boringssl-with-bazel/src/crypto/rand_extra/passive.c', - 'third_party/boringssl-with-bazel/src/crypto/rand_extra/rand_extra.c', - 'third_party/boringssl-with-bazel/src/crypto/rand_extra/trusty.c', - 'third_party/boringssl-with-bazel/src/crypto/rand_extra/windows.c', - 'third_party/boringssl-with-bazel/src/crypto/rc4/rc4.c', - 'third_party/boringssl-with-bazel/src/crypto/refcount.c', - 'third_party/boringssl-with-bazel/src/crypto/rsa_extra/rsa_asn1.c', - 'third_party/boringssl-with-bazel/src/crypto/rsa_extra/rsa_crypt.c', - 'third_party/boringssl-with-bazel/src/crypto/rsa_extra/rsa_print.c', - 'third_party/boringssl-with-bazel/src/crypto/siphash/siphash.c', - 'third_party/boringssl-with-bazel/src/crypto/spx/address.c', - 'third_party/boringssl-with-bazel/src/crypto/spx/fors.c', - 'third_party/boringssl-with-bazel/src/crypto/spx/merkle.c', - 'third_party/boringssl-with-bazel/src/crypto/spx/spx.c', - 'third_party/boringssl-with-bazel/src/crypto/spx/spx_util.c', - 'third_party/boringssl-with-bazel/src/crypto/spx/thash.c', - 'third_party/boringssl-with-bazel/src/crypto/spx/wots.c', - 'third_party/boringssl-with-bazel/src/crypto/stack/stack.c', - 'third_party/boringssl-with-bazel/src/crypto/thread.c', - 'third_party/boringssl-with-bazel/src/crypto/thread_none.c', - 'third_party/boringssl-with-bazel/src/crypto/thread_pthread.c', - 'third_party/boringssl-with-bazel/src/crypto/thread_win.c', - 'third_party/boringssl-with-bazel/src/crypto/trust_token/pmbtoken.c', - 'third_party/boringssl-with-bazel/src/crypto/trust_token/trust_token.c', - 'third_party/boringssl-with-bazel/src/crypto/trust_token/voprf.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/a_digest.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/a_sign.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/a_verify.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/algorithm.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/asn1_gen.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/by_dir.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/by_file.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/i2d_pr.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/name_print.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/policy.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/rsa_pss.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/t_crl.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/t_req.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/t_x509.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/t_x509a.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/v3_akey.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/v3_akeya.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/v3_alt.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/v3_bcons.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/v3_bitst.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/v3_conf.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/v3_cpols.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/v3_crld.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/v3_enum.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/v3_extku.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/v3_genn.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/v3_ia5.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/v3_info.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/v3_int.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/v3_lib.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/v3_ncons.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/v3_ocsp.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/v3_pcons.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/v3_pmaps.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/v3_prn.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/v3_purp.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/v3_skey.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/v3_utl.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/x509.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/x509_att.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/x509_cmp.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/x509_d2.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/x509_def.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/x509_ext.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/x509_lu.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/x509_obj.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/x509_req.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/x509_set.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/x509_trs.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/x509_txt.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/x509_v3.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/x509_vfy.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/x509_vpm.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/x509cset.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/x509name.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/x509rset.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/x509spki.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/x_algor.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/x_all.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/x_attrib.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/x_crl.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/x_exten.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/x_name.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/x_pubkey.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/x_req.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/x_sig.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/x_spki.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/x_val.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/x_x509.c', - 'third_party/boringssl-with-bazel/src/crypto/x509/x_x509a.c', - 'third_party/boringssl-with-bazel/src/ssl/bio_ssl.cc', - 'third_party/boringssl-with-bazel/src/ssl/d1_both.cc', - 'third_party/boringssl-with-bazel/src/ssl/d1_lib.cc', - 'third_party/boringssl-with-bazel/src/ssl/d1_pkt.cc', - 'third_party/boringssl-with-bazel/src/ssl/d1_srtp.cc', - 'third_party/boringssl-with-bazel/src/ssl/dtls_method.cc', - 'third_party/boringssl-with-bazel/src/ssl/dtls_record.cc', - 'third_party/boringssl-with-bazel/src/ssl/encrypted_client_hello.cc', - 'third_party/boringssl-with-bazel/src/ssl/extensions.cc', - 'third_party/boringssl-with-bazel/src/ssl/handoff.cc', - 'third_party/boringssl-with-bazel/src/ssl/handshake.cc', - 'third_party/boringssl-with-bazel/src/ssl/handshake_client.cc', - 'third_party/boringssl-with-bazel/src/ssl/handshake_server.cc', - 'third_party/boringssl-with-bazel/src/ssl/s3_both.cc', - 'third_party/boringssl-with-bazel/src/ssl/s3_lib.cc', - 'third_party/boringssl-with-bazel/src/ssl/s3_pkt.cc', - 'third_party/boringssl-with-bazel/src/ssl/ssl_aead_ctx.cc', - 'third_party/boringssl-with-bazel/src/ssl/ssl_asn1.cc', - 'third_party/boringssl-with-bazel/src/ssl/ssl_buffer.cc', - 'third_party/boringssl-with-bazel/src/ssl/ssl_cert.cc', - 'third_party/boringssl-with-bazel/src/ssl/ssl_cipher.cc', - 'third_party/boringssl-with-bazel/src/ssl/ssl_file.cc', - 'third_party/boringssl-with-bazel/src/ssl/ssl_key_share.cc', - 'third_party/boringssl-with-bazel/src/ssl/ssl_lib.cc', - 'third_party/boringssl-with-bazel/src/ssl/ssl_privkey.cc', - 'third_party/boringssl-with-bazel/src/ssl/ssl_session.cc', - 'third_party/boringssl-with-bazel/src/ssl/ssl_stat.cc', - 'third_party/boringssl-with-bazel/src/ssl/ssl_transcript.cc', - 'third_party/boringssl-with-bazel/src/ssl/ssl_versions.cc', - 'third_party/boringssl-with-bazel/src/ssl/ssl_x509.cc', - 'third_party/boringssl-with-bazel/src/ssl/t1_enc.cc', - 'third_party/boringssl-with-bazel/src/ssl/tls13_both.cc', - 'third_party/boringssl-with-bazel/src/ssl/tls13_client.cc', - 'third_party/boringssl-with-bazel/src/ssl/tls13_enc.cc', - 'third_party/boringssl-with-bazel/src/ssl/tls13_server.cc', - 'third_party/boringssl-with-bazel/src/ssl/tls_method.cc', - 'third_party/boringssl-with-bazel/src/ssl/tls_record.cc', - ], - }, - { - 'target_name': 'boringssl_test_util', - 'type': 'static_library', - 'dependencies': [ - ], - 'sources': [ - 'third_party/boringssl-with-bazel/src/crypto/test/abi_test.cc', - 'third_party/boringssl-with-bazel/src/crypto/test/file_test.cc', - 'third_party/boringssl-with-bazel/src/crypto/test/test_util.cc', - 'third_party/boringssl-with-bazel/src/crypto/test/wycheproof_util.cc', - ], - }, - { - 'target_name': 'benchmark', - 'type': 'static_library', - 'dependencies': [ - ], - 'sources': [ - 'third_party/benchmark/src/benchmark.cc', - 'third_party/benchmark/src/benchmark_api_internal.cc', - 'third_party/benchmark/src/benchmark_main.cc', - 'third_party/benchmark/src/benchmark_name.cc', - 'third_party/benchmark/src/benchmark_register.cc', - 'third_party/benchmark/src/benchmark_runner.cc', - 'third_party/benchmark/src/check.cc', - 'third_party/benchmark/src/colorprint.cc', - 'third_party/benchmark/src/commandlineflags.cc', - 'third_party/benchmark/src/complexity.cc', - 'third_party/benchmark/src/console_reporter.cc', - 'third_party/benchmark/src/counter.cc', - 'third_party/benchmark/src/csv_reporter.cc', - 'third_party/benchmark/src/json_reporter.cc', - 'third_party/benchmark/src/perf_counters.cc', - 'third_party/benchmark/src/reporter.cc', - 'third_party/benchmark/src/statistics.cc', - 'third_party/benchmark/src/string_util.cc', - 'third_party/benchmark/src/sysinfo.cc', - 'third_party/benchmark/src/timers.cc', - ], - }, - ] -} diff --git a/templates/grpc.gyp.template b/templates/grpc.gyp.template deleted file mode 100644 index 21b3f724da1..00000000000 --- a/templates/grpc.gyp.template +++ /dev/null @@ -1,162 +0,0 @@ -%YAML 1.2 ---- | - # GRPC GYP build file - - # This file has been automatically generated from a template file. - # Please look at the templates directory instead. - # This file can be regenerated from the template by running - # tools/buildgen/generate_projects.sh - - # Copyright 2015 gRPC authors. - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - <% - def is_absl_lib(target_name): - return target_name.startswith("absl/"); - %> - { - 'variables': { - # The openssl and zlib dependencies must be passed in as variables - # defined in an included gypi file, usually common.gypi. - 'openssl_gyp_target%': 'Please Define openssl_gyp_target variable', - 'zlib_gyp_target%': 'Please Define zlib_gyp_target variable', - - 'grpc_gcov%': 'false', - 'grpc_alpine%': 'false', - }, - 'target_defaults': { - 'configurations': { - % for name, args in configs.items(): - % if name in ['dbg', 'opt']: - '${{'dbg':'Debug', 'opt': 'Release'}[name]}': { - % for arg, prop in [('CPPFLAGS', 'cflags'), ('DEFINES', 'defines')]: - % if args.get(arg, None) is not None: - '${prop}': [ - % for item in args.get(arg).split(): - '${item}', - % endfor - ], - % endif - % endfor - }, - % endif - % endfor - }, - % for arg, prop in [('CPPFLAGS', 'cflags'), ('LDFLAGS', 'ldflags')]: - % if defaults['global'].get(arg, None) is not None: - '${prop}': [ - % for item in defaults['global'].get(arg).split(): - '${item}', - % endfor - ], - % endif - % endfor - 'cflags_c': [ - '-Werror', - '-std=c11', - ], - 'cflags_cc': [ - '-Werror', - '-std=c++14', - ], - 'include_dirs': [ - '.', - '../..', - 'include', - ], - 'defines': [ - 'GRPC_ARES=0', - ], - 'dependencies': [ - '<(openssl_gyp_target)', - '<(zlib_gyp_target)', - ], - 'conditions': [ - ['grpc_gcov=="true"', { - % for arg, prop in [('CPPFLAGS', 'cflags'), ('DEFINES', 'defines'), ('LDFLAGS', 'ldflags')]: - % if configs['gcov'].get(arg, None) is not None: - '${prop}': [ - % for item in configs['gcov'].get(arg).split(): - '${item}', - % endfor - ], - % endif - % endfor - }], - ['grpc_alpine=="true"', { - 'defines': [ - 'GPR_MUSL_LIBC_COMPAT' - ] - }], - ['OS == "win"', { - 'defines': [ - '_WIN32_WINNT=0x0600', - 'WIN32_LEAN_AND_MEAN', - '_HAS_EXCEPTIONS=0', - 'UNICODE', - '_UNICODE', - 'NOMINMAX', - ], - 'msvs_settings': { - 'VCCLCompilerTool': { - 'RuntimeLibrary': 1, # static debug - } - }, - "libraries": [ - "ws2_32" - ] - }], - ['OS == "mac"', { - 'xcode_settings': { - % if defaults['global'].get('CPPFLAGS', None) is not None: - 'OTHER_CFLAGS': [ - % for item in defaults['global'].get('CPPFLAGS').split(): - '${item}', - % endfor - ], - 'OTHER_CPLUSPLUSFLAGS': [ - % for item in defaults['global'].get('CPPFLAGS').split(): - '${item}', - % endfor - '-stdlib=libc++', - '-std=c++14', - '-Wno-error=deprecated-declarations', - ], - % endif - }, - }] - ] - }, - 'targets': [ - % for lib in libs: - % if getattr(lib, 'platforms', None) is None and lib.name != 'cares' and not is_absl_lib(lib.name): - { - 'target_name': '${lib.name}', - 'type': 'static_library', - 'dependencies': [ - % for dep in getattr(lib, 'deps', []): - % if dep != 'libssl': - '${dep}', - % endif - % endfor - ], - 'sources': [ - % for source in lib.src: - '${source}', - % endfor - ], - }, - % endif - % endfor - ] - } From c68e6c27ca54a8b7b98a21e6315c805f235d5dda Mon Sep 17 00:00:00 2001 From: Gregory Cooke Date: Fri, 15 Mar 2024 16:58:16 -0700 Subject: [PATCH 52/52] [Security Tests - Crl Provider] Fix inconsistent directory related tests (#36122) There were some failures in the Crl Directory related tests after https://github.com/grpc/grpc/pull/36031 This came down to https://github.com/grpc/grpc/pull/36031 adding some CRLs with bad qualities (invalid content/signatures, overriding issuer names) to the `test_creds/crl_data/crls` directory, which is used in the directory reloading tests. The tests began failing on some platforms because they were picking up these bad crls which were failing various checks, but the test was designed to assume that `test_creds/crl_data/crls` was a valid and good directory. This PR moves the bad CRLs to their own directory to prevent this accidental mash-up of test data. It also adds debug logging to our custom verification stack. Closes #36122 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/36122 from gtcooke94:crl_fix 508dd1370d85d84a52ac84c65525a0ed1ab8d4aa PiperOrigin-RevId: 616280898 --- src/core/tsi/ssl_transport_security.cc | 14 +++++++++---- src/core/tsi/ssl_transport_security_utils.cc | 11 ++++++++-- test/core/tsi/BUILD | 10 ++++----- .../tsi/crl_ssl_transport_security_test.cc | 7 ++++--- .../tsi/ssl_transport_security_utils_test.cc | 4 ++-- test/core/tsi/test_creds/crl_data/README | 3 ++- .../tsi/test_creds/crl_data/bad_crls/BUILD | 21 +++++++++++++++++++ .../crl_data/{crls => bad_crls}/evil.crl | 0 .../{crls => bad_crls}/invalid_content.crl | 0 .../{crls => bad_crls}/invalid_signature.crl | 0 test/core/tsi/test_creds/crl_data/crls/BUILD | 3 --- test/cpp/end2end/crl_provider_test.cc | 3 ++- 12 files changed, 55 insertions(+), 21 deletions(-) create mode 100644 test/core/tsi/test_creds/crl_data/bad_crls/BUILD rename test/core/tsi/test_creds/crl_data/{crls => bad_crls}/evil.crl (100%) rename test/core/tsi/test_creds/crl_data/{crls => bad_crls}/invalid_content.crl (100%) rename test/core/tsi/test_creds/crl_data/{crls => bad_crls}/invalid_signature.crl (100%) diff --git a/src/core/tsi/ssl_transport_security.cc b/src/core/tsi/ssl_transport_security.cc index 1ee9faf284a..17ad9077393 100644 --- a/src/core/tsi/ssl_transport_security.cc +++ b/src/core/tsi/ssl_transport_security.cc @@ -1045,10 +1045,12 @@ static bool ValidateCrl(X509* cert, X509* issuer, X509_CRL* crl) { // 6.3.3b verify issuer and scope valid = grpc_core::VerifyCrlCertIssuerNamesMatch(crl, cert); if (!valid) { + gpr_log(GPR_DEBUG, "CRL and cert issuer names mismatched."); return valid; } valid = grpc_core::HasCrlSignBit(issuer); if (!valid) { + gpr_log(GPR_DEBUG, "CRL issuer not allowed to sign CRLs."); return valid; } // 6.3.3c Not supporting deltas @@ -1058,6 +1060,9 @@ static bool ValidateCrl(X509* cert, X509* issuer, X509_CRL* crl) { // same. // 6.3.3g Verify CRL Signature valid = grpc_core::VerifyCrlSignature(crl, issuer); + if (!valid) { + gpr_log(GPR_DEBUG, "Crl signature check failed."); + } return valid; } @@ -1146,6 +1151,7 @@ static int CheckChainRevocation( static int CustomVerificationFunction(X509_STORE_CTX* ctx, void* arg) { int ret = X509_verify_cert(ctx); if (ret <= 0) { + gpr_log(GPR_DEBUG, "Failed to verify cert chain."); // Verification failed. We shouldn't expect to have a verified chain, so // there is no need to attempt to extract the root cert from it, check for // revocation, or check anything else. @@ -1154,10 +1160,10 @@ static int CustomVerificationFunction(X509_STORE_CTX* ctx, void* arg) { grpc_core::experimental::CrlProvider* provider = GetCrlProvider(ctx); if (provider != nullptr) { ret = CheckChainRevocation(ctx, provider); - } - if (ret <= 0) { - // Something has failed return the failure - return ret; + if (ret <= 0) { + gpr_log(GPR_DEBUG, "The chain failed revocation checks."); + return ret; + } } return RootCertExtractCallback(ctx, arg); } diff --git a/src/core/tsi/ssl_transport_security_utils.cc b/src/core/tsi/ssl_transport_security_utils.cc index 020e88fa0ac..1c05de9293d 100644 --- a/src/core/tsi/ssl_transport_security_utils.cc +++ b/src/core/tsi/ssl_transport_security_utils.cc @@ -259,12 +259,19 @@ bool VerifyCrlSignature(X509_CRL* crl, X509* issuer) { if (ikey == nullptr) { // Can't verify signature because we couldn't get the pubkey, fail the // check. + gpr_log(GPR_DEBUG, "Could not public key from certificate."); EVP_PKEY_free(ikey); return false; } - bool ret = X509_CRL_verify(crl, ikey) == 1; + int ret = X509_CRL_verify(crl, ikey); + if (ret < 0) { + gpr_log(GPR_DEBUG, + "There was an unexpected problem checking the CRL signature."); + } else if (ret == 0) { + gpr_log(GPR_DEBUG, "CRL failed verification."); + } EVP_PKEY_free(ikey); - return ret; + return ret == 1; } bool VerifyCrlCertIssuerNamesMatch(X509_CRL* crl, X509* cert) { diff --git a/test/core/tsi/BUILD b/test/core/tsi/BUILD index a4854f4dedf..fcf97c83eca 100644 --- a/test/core/tsi/BUILD +++ b/test/core/tsi/BUILD @@ -71,10 +71,10 @@ grpc_cc_test( "//test/core/tsi/test_creds/crl_data:evil_ca.pem", "//test/core/tsi/test_creds/crl_data:intermediate_ca.pem", "//test/core/tsi/test_creds/crl_data:leaf_signed_by_intermediate.pem", + "//test/core/tsi/test_creds/crl_data/bad_crls:invalid_content.crl", + "//test/core/tsi/test_creds/crl_data/bad_crls:invalid_signature.crl", "//test/core/tsi/test_creds/crl_data/crls:current.crl", "//test/core/tsi/test_creds/crl_data/crls:intermediate.crl", - "//test/core/tsi/test_creds/crl_data/crls:invalid_content.crl", - "//test/core/tsi/test_creds/crl_data/crls:invalid_signature.crl", ], external_deps = ["gtest"], language = "C++", @@ -136,13 +136,13 @@ grpc_cc_test( "//test/core/tsi/test_creds/crl_data:revoked.pem", "//test/core/tsi/test_creds/crl_data:valid.key", "//test/core/tsi/test_creds/crl_data:valid.pem", + "//test/core/tsi/test_creds/crl_data/bad_crls:evil.crl", + "//test/core/tsi/test_creds/crl_data/bad_crls:invalid_content.crl", + "//test/core/tsi/test_creds/crl_data/bad_crls:invalid_signature.crl", "//test/core/tsi/test_creds/crl_data/crls:ab06acdd.r0", "//test/core/tsi/test_creds/crl_data/crls:b9322cac.r0", "//test/core/tsi/test_creds/crl_data/crls:current.crl", - "//test/core/tsi/test_creds/crl_data/crls:evil.crl", "//test/core/tsi/test_creds/crl_data/crls:intermediate.crl", - "//test/core/tsi/test_creds/crl_data/crls:invalid_content.crl", - "//test/core/tsi/test_creds/crl_data/crls:invalid_signature.crl", "//test/core/tsi/test_creds/crl_data/crls_missing_intermediate:ab06acdd.r0", "//test/core/tsi/test_creds/crl_data/crls_missing_root:b9322cac.r0", ], diff --git a/test/core/tsi/crl_ssl_transport_security_test.cc b/test/core/tsi/crl_ssl_transport_security_test.cc index da873e80956..93018a818d0 100644 --- a/test/core/tsi/crl_ssl_transport_security_test.cc +++ b/test/core/tsi/crl_ssl_transport_security_test.cc @@ -69,10 +69,11 @@ const char* kRootCrlPath = "test/core/tsi/test_creds/crl_data/crls/current.crl"; const char* kIntermediateCrlPath = "test/core/tsi/test_creds/crl_data/crls/intermediate.crl"; const char* kModifiedSignaturePath = - "test/core/tsi/test_creds/crl_data/crls/invalid_signature.crl"; + "test/core/tsi/test_creds/crl_data/bad_crls/invalid_signature.crl"; const char* kModifiedContentPath = - "test/core/tsi/test_creds/crl_data/crls/invalid_content.crl"; -const char* kEvilCrlPath = "test/core/tsi/test_creds/crl_data/crls/evil.crl"; + "test/core/tsi/test_creds/crl_data/bad_crls/invalid_content.crl"; +const char* kEvilCrlPath = + "test/core/tsi/test_creds/crl_data/bad_crls/evil.crl"; class CrlSslTransportSecurityTest : public testing::TestWithParam { diff --git a/test/core/tsi/ssl_transport_security_utils_test.cc b/test/core/tsi/ssl_transport_security_utils_test.cc index 54e705d7574..2b7b6dfb36f 100644 --- a/test/core/tsi/ssl_transport_security_utils_test.cc +++ b/test/core/tsi/ssl_transport_security_utils_test.cc @@ -46,9 +46,9 @@ namespace testing { const char* kValidCrl = "test/core/tsi/test_creds/crl_data/crls/current.crl"; const char* kCrlIssuer = "test/core/tsi/test_creds/crl_data/ca.pem"; const char* kModifiedSignature = - "test/core/tsi/test_creds/crl_data/crls/invalid_signature.crl"; + "test/core/tsi/test_creds/crl_data/bad_crls/invalid_signature.crl"; const char* kModifiedContent = - "test/core/tsi/test_creds/crl_data/crls/invalid_content.crl"; + "test/core/tsi/test_creds/crl_data/bad_crls/invalid_content.crl"; const char* kIntermediateCrl = "test/core/tsi/test_creds/crl_data/crls/intermediate.crl"; const char* kIntermediateCrlIssuer = diff --git a/test/core/tsi/test_creds/crl_data/README b/test/core/tsi/test_creds/crl_data/README index f69fd1b75b3..3e5cfab739f 100644 --- a/test/core/tsi/test_creds/crl_data/README +++ b/test/core/tsi/test_creds/crl_data/README @@ -57,7 +57,8 @@ Run `ca_with_akid_gen.sh` from the `test/core/tsi/test_creds/crl_data` directory Create CRLs with modified signatures and content ---------------------------------------------------------------------------- -Make two copies of `test/core/tsi/test_creds/crl_data/crls/current.crl` named `test/core/tsi/test_creds/crl_data/crls/invalid_content.crl` and `test/core/tsi/test_creds/crl_data/crls/invalid_signature.crl`. +Make a directory `test/core/tsi/test_creds/crl_data/bad_crls +Make two copies of `test/core/tsi/test_creds/crl_data/crls/current.crl` in the bad_crls directory named `test/core/tsi/test_creds/crl_data/bad_crls/invalid_content.crl` and `test/core/tsi/test_creds/crl_data/bad_crls/invalid_signature.crl`. In `invalid_content.crl`, change the first letter on the second line. In `invalid_signature.crl`, change the last letter before the `=` on the second to last line. diff --git a/test/core/tsi/test_creds/crl_data/bad_crls/BUILD b/test/core/tsi/test_creds/crl_data/bad_crls/BUILD new file mode 100644 index 00000000000..f74dabcf8a0 --- /dev/null +++ b/test/core/tsi/test_creds/crl_data/bad_crls/BUILD @@ -0,0 +1,21 @@ +# Copyright 2021 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +licenses(["notice"]) + +exports_files([ + "evil.crl", + "invalid_content.crl", + "invalid_signature.crl", +]) diff --git a/test/core/tsi/test_creds/crl_data/crls/evil.crl b/test/core/tsi/test_creds/crl_data/bad_crls/evil.crl similarity index 100% rename from test/core/tsi/test_creds/crl_data/crls/evil.crl rename to test/core/tsi/test_creds/crl_data/bad_crls/evil.crl diff --git a/test/core/tsi/test_creds/crl_data/crls/invalid_content.crl b/test/core/tsi/test_creds/crl_data/bad_crls/invalid_content.crl similarity index 100% rename from test/core/tsi/test_creds/crl_data/crls/invalid_content.crl rename to test/core/tsi/test_creds/crl_data/bad_crls/invalid_content.crl diff --git a/test/core/tsi/test_creds/crl_data/crls/invalid_signature.crl b/test/core/tsi/test_creds/crl_data/bad_crls/invalid_signature.crl similarity index 100% rename from test/core/tsi/test_creds/crl_data/crls/invalid_signature.crl rename to test/core/tsi/test_creds/crl_data/bad_crls/invalid_signature.crl diff --git a/test/core/tsi/test_creds/crl_data/crls/BUILD b/test/core/tsi/test_creds/crl_data/crls/BUILD index e1a36f42c8f..922d279aa6a 100644 --- a/test/core/tsi/test_creds/crl_data/crls/BUILD +++ b/test/core/tsi/test_creds/crl_data/crls/BUILD @@ -19,7 +19,4 @@ exports_files([ "b9322cac.r0", "current.crl", "intermediate.crl", - "invalid_signature.crl", - "invalid_content.crl", - "evil.crl", ]) diff --git a/test/cpp/end2end/crl_provider_test.cc b/test/cpp/end2end/crl_provider_test.cc index 6902fd7443e..28dabae37d3 100644 --- a/test/cpp/end2end/crl_provider_test.cc +++ b/test/cpp/end2end/crl_provider_test.cc @@ -61,7 +61,8 @@ const char* kRevokedCertPath = "test/core/tsi/test_creds/crl_data/revoked.pem"; const char* kValidKeyPath = "test/core/tsi/test_creds/crl_data/valid.key"; const char* kValidCertPath = "test/core/tsi/test_creds/crl_data/valid.pem"; const char* kRootCrlPath = "test/core/tsi/test_creds/crl_data/crls/current.crl"; -const char* kCrlDirectoryPath = "test/core/tsi/test_creds/crl_data/crls/"; +const char* kCrlDirectoryPath = + "test/core/tsi/test_creds/crl_data/crl_provider_test_dir/"; constexpr char kMessage[] = "Hello"; // This test must be at the top of the file because the