From 3459e5c7b83574ad21c99f0df96dc1e1a7f04581 Mon Sep 17 00:00:00 2001 From: Yousuk Seung Date: Mon, 12 Feb 2024 10:25:13 -0800 Subject: [PATCH 01/37] [work-serializer] deflake tests with dispatching workserializer PiperOrigin-RevId: 606292107 --- test/core/gprpp/work_serializer_test.cc | 9 +++++++-- test/core/xds/xds_client_test.cc | 11 ++++++----- test/core/xds/xds_transport_fake.cc | 6 ++++-- test/core/xds/xds_transport_fake.h | 13 ++++++++++--- 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/test/core/gprpp/work_serializer_test.cc b/test/core/gprpp/work_serializer_test.cc index 918c845e153..3f8be452338 100644 --- a/test/core/gprpp/work_serializer_test.cc +++ b/test/core/gprpp/work_serializer_test.cc @@ -74,9 +74,11 @@ TEST(WorkSerializerTest, ExecuteOneScheduleAndDrain) { gpr_event done; gpr_event_init(&done); lock->Schedule( - [&done]() { gpr_event_set(&done, reinterpret_cast(1)); }, + [&done]() { + EXPECT_EQ(gpr_event_get(&done), nullptr); + gpr_event_set(&done, reinterpret_cast(1)); + }, DEBUG_LOCATION); - EXPECT_EQ(gpr_event_get(&done), nullptr); lock->DrainQueue(); EXPECT_TRUE(gpr_event_wait(&done, grpc_timeout_seconds_to_deadline(5)) != nullptr); @@ -288,6 +290,9 @@ TEST(WorkSerializerTest, MetricsWork) { auto before = global_stats().Collect(); auto stats_diff_from = [&before](absl::AnyInvocable f) { f(); + // Insert a pause for the work serialier to update the stats. Reading stats + // here can still race with the work serializer's update attempt. + gpr_sleep_until(grpc_timeout_seconds_to_deadline(1)); auto after = global_stats().Collect(); auto diff = after->Diff(*before); before = std::move(after); diff --git a/test/core/xds/xds_client_test.cc b/test/core/xds/xds_client_test.cc index 3f9e2bb9615..7a699bcfcb0 100644 --- a/test/core/xds/xds_client_test.cc +++ b/test/core/xds/xds_client_test.cc @@ -2707,6 +2707,7 @@ TEST_F(XdsClientTest, FederationChannelFailureReportedToWatchers) { } TEST_F(XdsClientTest, AdsReadWaitsForHandleRelease) { + const absl::Duration timeout = absl::Seconds(5) * grpc_test_slowdown_factor(); InitXdsClient(); // Start watches for "foo1" and "foo2". auto watcher1 = StartFooWatch("foo1"); @@ -2759,11 +2760,11 @@ TEST_F(XdsClientTest, AdsReadWaitsForHandleRelease) { /*version_info=*/"1", /*response_nonce=*/"A", /*error_detail=*/absl::OkStatus(), /*resource_names=*/{"foo1", "foo2"}); - EXPECT_EQ(stream->reads_started(), 1); + EXPECT_TRUE(stream->WaitForReadsStarted(1, timeout)); resource1->read_delay_handle.reset(); - EXPECT_EQ(stream->reads_started(), 1); + EXPECT_TRUE(stream->WaitForReadsStarted(1, timeout)); resource2->read_delay_handle.reset(); - EXPECT_EQ(stream->reads_started(), 2); + EXPECT_TRUE(stream->WaitForReadsStarted(2, timeout)); resource1 = watcher1->WaitForNextResourceAndHandle(); ASSERT_NE(resource1, absl::nullopt); EXPECT_EQ(resource1->resource->name, "foo1"); @@ -2776,9 +2777,9 @@ TEST_F(XdsClientTest, AdsReadWaitsForHandleRelease) { /*version_info=*/"2", /*response_nonce=*/"B", /*error_detail=*/absl::OkStatus(), /*resource_names=*/{"foo1", "foo2"}); - EXPECT_EQ(stream->reads_started(), 2); + EXPECT_TRUE(stream->WaitForReadsStarted(2, timeout)); resource1->read_delay_handle.reset(); - EXPECT_EQ(stream->reads_started(), 3); + EXPECT_TRUE(stream->WaitForReadsStarted(3, timeout)); // Cancel watch. CancelFooWatch(watcher1.get(), "foo1"); request = WaitForRequest(stream.get()); diff --git a/test/core/xds/xds_transport_fake.cc b/test/core/xds/xds_transport_fake.cc index bc1134f08b6..1153d870151 100644 --- a/test/core/xds/xds_transport_fake.cc +++ b/test/core/xds/xds_transport_fake.cc @@ -81,7 +81,7 @@ void FakeXdsTransportFactory::FakeStreamingCall::SendMessage( MutexLock lock(&mu_); GPR_ASSERT(!orphaned_); from_client_messages_.push_back(std::move(payload)); - cv_.Signal(); + cv_client_msg_.Signal(); if (transport_->auto_complete_messages_from_client()) { CompleteSendMessageFromClientLocked(/*ok=*/true); } @@ -97,7 +97,8 @@ FakeXdsTransportFactory::FakeStreamingCall::WaitForMessageFromClient( absl::Duration timeout) { MutexLock lock(&mu_); while (from_client_messages_.empty()) { - if (cv_.WaitWithTimeout(&mu_, timeout * grpc_test_slowdown_factor())) { + if (cv_client_msg_.WaitWithTimeout(&mu_, + timeout * grpc_test_slowdown_factor())) { return absl::nullopt; } } @@ -133,6 +134,7 @@ void FakeXdsTransportFactory::FakeStreamingCall::StartRecvMessage() { } ++reads_started_; ++num_pending_reads_; + cv_reads_started_.SignalAll(); if (!to_client_messages_.empty()) { // Dispatch pending message (if there's one) on a separate thread to avoid // recursion diff --git a/test/core/xds/xds_transport_fake.h b/test/core/xds/xds_transport_fake.h index c9b24a536ba..a3f50caae25 100644 --- a/test/core/xds/xds_transport_fake.h +++ b/test/core/xds/xds_transport_fake.h @@ -88,9 +88,15 @@ class FakeXdsTransportFactory : public XdsTransportFactory { bool Orphaned(); - size_t reads_started() { + bool WaitForReadsStarted(size_t expected, absl::Duration timeout) { MutexLock lock(&mu_); - return reads_started_; + const absl::Time deadline = absl::Now() + timeout; + do { + if (reads_started_ == expected) { + return true; + } + } while (!cv_reads_started_.WaitWithDeadline(&mu_, deadline)); + return false; } private: @@ -122,7 +128,8 @@ class FakeXdsTransportFactory : public XdsTransportFactory { const char* method_; Mutex mu_; - CondVar cv_; + CondVar cv_reads_started_; + CondVar cv_client_msg_; RefCountedPtr event_handler_ ABSL_GUARDED_BY(&mu_); std::deque from_client_messages_ ABSL_GUARDED_BY(&mu_); bool status_sent_ ABSL_GUARDED_BY(&mu_) = false; From 2b3f21daea36e26151c7a46696fd3b0689b3ecf9 Mon Sep 17 00:00:00 2001 From: gRPC Team Bot Date: Mon, 12 Feb 2024 14:09:16 -0800 Subject: [PATCH 02/37] Internal change PiperOrigin-RevId: 606360614 --- test/core/transport/chaotic_good/BUILD | 5 ++++- test/core/transport/chttp2/BUILD | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/test/core/transport/chaotic_good/BUILD b/test/core/transport/chaotic_good/BUILD index 3b47bf92aef..f76d7c35041 100644 --- a/test/core/transport/chaotic_good/BUILD +++ b/test/core/transport/chaotic_good/BUILD @@ -67,7 +67,10 @@ grpc_fuzzer( external_deps = ["absl/status:statusor"], language = "C++", tags = ["no_windows"], - deps = ["//src/core:chaotic_good_frame_header"], + deps = [ + "//:grpc", + "//src/core:chaotic_good_frame_header", + ], ) grpc_cc_test( diff --git a/test/core/transport/chttp2/BUILD b/test/core/transport/chttp2/BUILD index 1c11d2f5548..8aa43fc8c96 100644 --- a/test/core/transport/chttp2/BUILD +++ b/test/core/transport/chttp2/BUILD @@ -99,6 +99,7 @@ grpc_fuzzer( language = "C++", tags = ["no_windows"], deps = [ + "//:grpc", "//src/core:decode_huff", "//src/core:huffsyms", ], From c75d5c98782d07b6b29cc0cc132bb503e8eb7843 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 12 Feb 2024 14:16:13 -0800 Subject: [PATCH 03/37] [chaotic-good] Credential implementation (#35884) Closes #35884 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/35884 from ctiller:noodling dbba5d5578119f7baeec7df3747f91e8675e4574 PiperOrigin-RevId: 606362603 --- BUILD | 18 +++++++++ bazel/grpc_build_system.bzl | 1 + src/cpp/ext/chaotic_good.cc | 80 +++++++++++++++++++++++++++++++++++++ src/cpp/ext/chaotic_good.h | 30 ++++++++++++++ 4 files changed, 129 insertions(+) create mode 100644 src/cpp/ext/chaotic_good.cc create mode 100644 src/cpp/ext/chaotic_good.h diff --git a/BUILD b/BUILD index a02606d4ff0..6bb4e6b2b4f 100644 --- a/BUILD +++ b/BUILD @@ -4140,6 +4140,24 @@ grpc_cc_library( ], ) +grpc_cc_library( + name = "grpcpp_chaotic_good", + srcs = [ + "src/cpp/ext/chaotic_good.cc", + ], + hdrs = [ + "src/cpp/ext/chaotic_good.h", + ], + visibility = ["@grpc:chaotic_good"], + deps = [ + "gpr", + "grpc++_public_hdrs", + "grpc_public_hdrs", + "//src/core:chaotic_good_connector", + "//src/core:chaotic_good_server", + ], +) + grpc_cc_library( name = "subprocess", srcs = [ diff --git a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl index aa1b6645f7c..6e09896b4ed 100644 --- a/bazel/grpc_build_system.bzl +++ b/bazel/grpc_build_system.bzl @@ -95,6 +95,7 @@ def _update_visibility(visibility): "alt_grpc++_base_unsecure_legacy": PRIVATE, "alts_frame_protector": PRIVATE, "channelz": PRIVATE, + "chaotic_good": PRIVATE, "client_channel": PRIVATE, "cli": PRIVATE, "debug_location": PRIVATE, diff --git a/src/cpp/ext/chaotic_good.cc b/src/cpp/ext/chaotic_good.cc new file mode 100644 index 00000000000..8711a73ace0 --- /dev/null +++ b/src/cpp/ext/chaotic_good.cc @@ -0,0 +1,80 @@ +// Copyright 2024 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/cpp/ext/chaotic_good.h" + +#include + +#include + +#include "src/core/ext/transport/chaotic_good/client/chaotic_good_connector.h" +#include "src/core/ext/transport/chaotic_good/server/chaotic_good_server.h" +#include "src/core/lib/gprpp/crash.h" + +namespace grpc { + +namespace { + +class ChaoticGoodInsecureChannelCredentialsImpl final + : public ChannelCredentials { + public: + std::shared_ptr CreateChannelImpl( + const grpc::string& target, const grpc::ChannelArguments& args) override { + return CreateChannelWithInterceptors(target, args, {}); + } + + std::shared_ptr CreateChannelWithInterceptors( + const grpc::string& target, const grpc::ChannelArguments& args, + std::vector< + std::unique_ptr> + interceptor_creators) override { + grpc_channel_args channel_args; + args.SetChannelArgs(&channel_args); + auto channel = grpc::CreateChannelInternal( + "", grpc_chaotic_good_channel_create(target.c_str(), &channel_args), + std::move(interceptor_creators)); + grpc_channel_args_destroy(&channel_args); + return channel; + } + + SecureChannelCredentials* AsSecureCredentials() override { return nullptr; } + + private: + bool IsInsecure() const override { return true; } +}; + +class ChaoticGoodInsecureServerCredentialsImpl final + : public ServerCredentials { + public: + int AddPortToServer(const std::string& addr, grpc_server* server) override { + return grpc_server_add_chaotic_good_port(server, addr.c_str()); + } + + void SetAuthMetadataProcessor( + const std::shared_ptr&) override { + grpc_core::Crash("Not supported on insecure server credentials"); + } +}; + +} // namespace + +std::shared_ptr ChaoticGoodInsecureChannelCredentials() { + return std::make_shared(); +} + +std::shared_ptr ChaoticGoodInsecureServerCredentials() { + return std::make_shared(); +} + +} // namespace grpc diff --git a/src/cpp/ext/chaotic_good.h b/src/cpp/ext/chaotic_good.h new file mode 100644 index 00000000000..b203f473e59 --- /dev/null +++ b/src/cpp/ext/chaotic_good.h @@ -0,0 +1,30 @@ +// Copyright 2024 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GRPC_SRC_CPP_EXT_CHAOTIC_GOOD_H +#define GRPC_SRC_CPP_EXT_CHAOTIC_GOOD_H + +#include + +#include +#include + +namespace grpc { + +std::shared_ptr ChaoticGoodInsecureChannelCredentials(); +std::shared_ptr ChaoticGoodInsecureServerCredentials(); + +} // namespace grpc + +#endif // GRPC_SRC_CPP_EXT_CHAOTIC_GOOD_H From 6df52f865feec8de0dffe16e5b62ec01741a55b6 Mon Sep 17 00:00:00 2001 From: Eugene Ostroukhov Date: Mon, 12 Feb 2024 15:25:16 -0800 Subject: [PATCH 04/37] [test] Increase test timeout (#35887) Closes #35887 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/35887 from eugeneo:release/162 ed2f0f08560f2fb89cb9a46c02c98d09c299d15d PiperOrigin-RevId: 606382703 --- tools/run_tests/artifacts/distribtest_targets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/artifacts/distribtest_targets.py b/tools/run_tests/artifacts/distribtest_targets.py index cff61987d04..fe71900756f 100644 --- a/tools/run_tests/artifacts/distribtest_targets.py +++ b/tools/run_tests/artifacts/distribtest_targets.py @@ -303,7 +303,7 @@ class PHP7DistribTest(object): self.name, ["test/distrib/php/run_distrib_test_macos.sh"], environ={"EXTERNAL_GIT_ROOT": "../../../.."}, - timeout_seconds=20 * 60, + timeout_seconds=30 * 60, use_workspace=True, ) else: From 86d4155fe4c27ae18fac6f09c51056403f63183d Mon Sep 17 00:00:00 2001 From: Eugene Ostroukhov Date: Tue, 13 Feb 2024 08:47:05 -0800 Subject: [PATCH 05/37] [Release] Bump core version to 39.0.0 for upcoming release (#35889) Change was created by the release automation script. See go/grpc-release. Closes #35889 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/35889 from eugeneo:bump_core_version_202402121600 dfab90395b8e486dc797ce25090e1ddbf2590da6 PiperOrigin-RevId: 606635870 --- BUILD | 2 +- CMakeLists.txt | 4 +-- Makefile | 48 ++++++++++++++-------------- build_config.rb | 2 +- build_handwritten.yaml | 2 +- src/core/lib/surface/version.cc | 2 +- src/objective-c/tests/version.h | 2 +- tools/doxygen/Doxyfile.core | 2 +- tools/doxygen/Doxyfile.core.internal | 2 +- 9 files changed, 33 insertions(+), 33 deletions(-) diff --git a/BUILD b/BUILD index 6bb4e6b2b4f..455b7db1705 100644 --- a/BUILD +++ b/BUILD @@ -213,7 +213,7 @@ python_config_settings() # This should be updated along with build_handwritten.yaml g_stands_for = "guardian" # @unused -core_version = "38.0.0" # @unused +core_version = "39.0.0" # @unused version = "1.62.0-dev" # @unused diff --git a/CMakeLists.txt b/CMakeLists.txt index fa742616c6a..fe9b7314195 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,8 +26,8 @@ cmake_minimum_required(VERSION 3.13) set(PACKAGE_NAME "grpc") set(PACKAGE_VERSION "1.62.0-dev") -set(gRPC_CORE_VERSION "38.0.0") -set(gRPC_CORE_SOVERSION "38") +set(gRPC_CORE_VERSION "39.0.0") +set(gRPC_CORE_SOVERSION "39") set(gRPC_CPP_VERSION "1.62.0-dev") set(gRPC_CPP_SOVERSION "1.62") set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}") diff --git a/Makefile b/Makefile index 59acce49c60..e268990e44b 100644 --- a/Makefile +++ b/Makefile @@ -410,7 +410,7 @@ E = @echo Q = @ endif -CORE_VERSION = 38.0.0 +CORE_VERSION = 39.0.0 CPP_VERSION = 1.62.0-dev CPPFLAGS_NO_ARCH += $(addprefix -I, $(INCLUDES)) $(addprefix -D, $(DEFINES)) @@ -447,7 +447,7 @@ SHARED_EXT_CORE = dll SHARED_EXT_CPP = dll SHARED_PREFIX = -SHARED_VERSION_CORE = -38 +SHARED_VERSION_CORE = -39 SHARED_VERSION_CPP = -1 else ifeq ($(SYSTEM),Darwin) EXECUTABLE_SUFFIX = @@ -818,8 +818,8 @@ $(LIBDIR)/$(CONFIG)/libaddress_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): ifeq ($(SYSTEM),Darwin) $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)address_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libaddress_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBADDRESS_SORTING_OBJS) $(LDLIBS) else - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libaddress_sorting.so.38 -o $(LIBDIR)/$(CONFIG)/libaddress_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBADDRESS_SORTING_OBJS) $(LDLIBS) - $(Q) ln -sf $(SHARED_PREFIX)address_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libaddress_sorting$(SHARED_VERSION_CORE).so.38 + $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libaddress_sorting.so.39 -o $(LIBDIR)/$(CONFIG)/libaddress_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBADDRESS_SORTING_OBJS) $(LDLIBS) + $(Q) ln -sf $(SHARED_PREFIX)address_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libaddress_sorting$(SHARED_VERSION_CORE).so.39 $(Q) ln -sf $(SHARED_PREFIX)address_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libaddress_sorting$(SHARED_VERSION_CORE).so endif endif @@ -943,8 +943,8 @@ $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBGPR_OB ifeq ($(SYSTEM),Darwin) $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGPR_OBJS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) else - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgpr.so.38 -o $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGPR_OBJS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) - $(Q) ln -sf $(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).so.38 + $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgpr.so.39 -o $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGPR_OBJS) $(GRPC_ABSEIL_MERGE_LIBS) $(LDLIBS) + $(Q) ln -sf $(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).so.39 $(Q) ln -sf $(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).so endif endif @@ -1871,8 +1871,8 @@ $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBGRPC_ ifeq ($(SYSTEM),Darwin) $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_OBJS) $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libgpr.a $(GRPC_ABSEIL_MERGE_LIBS) $(LIBDIR)/$(CONFIG)/libcares.a $(ZLIB_MERGE_LIBS) $(LIBDIR)/$(CONFIG)/libre2.a $(LIBDIR)/$(CONFIG)/libupb_textformat_lib.a $(LIBDIR)/$(CONFIG)/libupb_json_lib.a $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(OPENSSL_MERGE_LIBS) $(LDLIBS_SECURE) $(LDLIBS) else - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgrpc.so.38 -o $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_OBJS) $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libgpr.a $(GRPC_ABSEIL_MERGE_LIBS) $(LIBDIR)/$(CONFIG)/libcares.a $(ZLIB_MERGE_LIBS) $(LIBDIR)/$(CONFIG)/libre2.a $(LIBDIR)/$(CONFIG)/libupb_textformat_lib.a $(LIBDIR)/$(CONFIG)/libupb_json_lib.a $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(OPENSSL_MERGE_LIBS) $(LDLIBS_SECURE) $(LDLIBS) - $(Q) ln -sf $(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).so.38 + $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgrpc.so.39 -o $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_OBJS) $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libgpr.a $(GRPC_ABSEIL_MERGE_LIBS) $(LIBDIR)/$(CONFIG)/libcares.a $(ZLIB_MERGE_LIBS) $(LIBDIR)/$(CONFIG)/libre2.a $(LIBDIR)/$(CONFIG)/libupb_textformat_lib.a $(LIBDIR)/$(CONFIG)/libupb_json_lib.a $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(OPENSSL_MERGE_LIBS) $(LDLIBS_SECURE) $(LDLIBS) + $(Q) ln -sf $(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).so.39 $(Q) ln -sf $(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).so endif endif @@ -2400,8 +2400,8 @@ $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $ ifeq ($(SYSTEM),Darwin) $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libgpr.a $(GRPC_ABSEIL_MERGE_LIBS) $(LIBDIR)/$(CONFIG)/libcares.a $(ZLIB_MERGE_LIBS) $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(LDLIBS) else - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgrpc_unsecure.so.38 -o $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libgpr.a $(GRPC_ABSEIL_MERGE_LIBS) $(LIBDIR)/$(CONFIG)/libcares.a $(ZLIB_MERGE_LIBS) $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(LDLIBS) - $(Q) ln -sf $(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).so.38 + $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgrpc_unsecure.so.39 -o $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libgpr.a $(GRPC_ABSEIL_MERGE_LIBS) $(LIBDIR)/$(CONFIG)/libcares.a $(ZLIB_MERGE_LIBS) $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(LDLIBS) + $(Q) ln -sf $(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).so.39 $(Q) ln -sf $(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).so endif endif @@ -2467,8 +2467,8 @@ $(LIBDIR)/$(CONFIG)/libre2$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBRE2_OB ifeq ($(SYSTEM),Darwin) $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)re2$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libre2$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBRE2_OBJS) $(LDLIBS) else - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libre2.so.38 -o $(LIBDIR)/$(CONFIG)/libre2$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBRE2_OBJS) $(LDLIBS) - $(Q) ln -sf $(SHARED_PREFIX)re2$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libre2$(SHARED_VERSION_CORE).so.38 + $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libre2.so.39 -o $(LIBDIR)/$(CONFIG)/libre2$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBRE2_OBJS) $(LDLIBS) + $(Q) ln -sf $(SHARED_PREFIX)re2$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libre2$(SHARED_VERSION_CORE).so.39 $(Q) ln -sf $(SHARED_PREFIX)re2$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libre2$(SHARED_VERSION_CORE).so endif endif @@ -2513,8 +2513,8 @@ $(LIBDIR)/$(CONFIG)/libupb_base_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $( ifeq ($(SYSTEM),Darwin) $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)upb_base_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libupb_base_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUPB_BASE_LIB_OBJS) $(LDLIBS) else - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libupb_base_lib.so.38 -o $(LIBDIR)/$(CONFIG)/libupb_base_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUPB_BASE_LIB_OBJS) $(LDLIBS) - $(Q) ln -sf $(SHARED_PREFIX)upb_base_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libupb_base_lib$(SHARED_VERSION_CORE).so.38 + $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libupb_base_lib.so.39 -o $(LIBDIR)/$(CONFIG)/libupb_base_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUPB_BASE_LIB_OBJS) $(LDLIBS) + $(Q) ln -sf $(SHARED_PREFIX)upb_base_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libupb_base_lib$(SHARED_VERSION_CORE).so.39 $(Q) ln -sf $(SHARED_PREFIX)upb_base_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libupb_base_lib$(SHARED_VERSION_CORE).so endif endif @@ -2593,8 +2593,8 @@ $(LIBDIR)/$(CONFIG)/libupb_json_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $( ifeq ($(SYSTEM),Darwin) $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)upb_json_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libupb_json_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUPB_JSON_LIB_OBJS) $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(LDLIBS) else - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libupb_json_lib.so.38 -o $(LIBDIR)/$(CONFIG)/libupb_json_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUPB_JSON_LIB_OBJS) $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(LDLIBS) - $(Q) ln -sf $(SHARED_PREFIX)upb_json_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libupb_json_lib$(SHARED_VERSION_CORE).so.38 + $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libupb_json_lib.so.39 -o $(LIBDIR)/$(CONFIG)/libupb_json_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUPB_JSON_LIB_OBJS) $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(LDLIBS) + $(Q) ln -sf $(SHARED_PREFIX)upb_json_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libupb_json_lib$(SHARED_VERSION_CORE).so.39 $(Q) ln -sf $(SHARED_PREFIX)upb_json_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libupb_json_lib$(SHARED_VERSION_CORE).so endif endif @@ -2640,8 +2640,8 @@ $(LIBDIR)/$(CONFIG)/libupb_mem_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(L ifeq ($(SYSTEM),Darwin) $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)upb_mem_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libupb_mem_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUPB_MEM_LIB_OBJS) $(LDLIBS) else - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libupb_mem_lib.so.38 -o $(LIBDIR)/$(CONFIG)/libupb_mem_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUPB_MEM_LIB_OBJS) $(LDLIBS) - $(Q) ln -sf $(SHARED_PREFIX)upb_mem_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libupb_mem_lib$(SHARED_VERSION_CORE).so.38 + $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libupb_mem_lib.so.39 -o $(LIBDIR)/$(CONFIG)/libupb_mem_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUPB_MEM_LIB_OBJS) $(LDLIBS) + $(Q) ln -sf $(SHARED_PREFIX)upb_mem_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libupb_mem_lib$(SHARED_VERSION_CORE).so.39 $(Q) ln -sf $(SHARED_PREFIX)upb_mem_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libupb_mem_lib$(SHARED_VERSION_CORE).so endif endif @@ -2693,8 +2693,8 @@ $(LIBDIR)/$(CONFIG)/libupb_message_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): ifeq ($(SYSTEM),Darwin) $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)upb_message_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libupb_message_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUPB_MESSAGE_LIB_OBJS) $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(LDLIBS) else - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libupb_message_lib.so.38 -o $(LIBDIR)/$(CONFIG)/libupb_message_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUPB_MESSAGE_LIB_OBJS) $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(LDLIBS) - $(Q) ln -sf $(SHARED_PREFIX)upb_message_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libupb_message_lib$(SHARED_VERSION_CORE).so.38 + $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libupb_message_lib.so.39 -o $(LIBDIR)/$(CONFIG)/libupb_message_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUPB_MESSAGE_LIB_OBJS) $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(LDLIBS) + $(Q) ln -sf $(SHARED_PREFIX)upb_message_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libupb_message_lib$(SHARED_VERSION_CORE).so.39 $(Q) ln -sf $(SHARED_PREFIX)upb_message_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libupb_message_lib$(SHARED_VERSION_CORE).so endif endif @@ -2772,8 +2772,8 @@ $(LIBDIR)/$(CONFIG)/libupb_textformat_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_COR ifeq ($(SYSTEM),Darwin) $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)upb_textformat_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libupb_textformat_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUPB_TEXTFORMAT_LIB_OBJS) $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(LDLIBS) else - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libupb_textformat_lib.so.38 -o $(LIBDIR)/$(CONFIG)/libupb_textformat_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUPB_TEXTFORMAT_LIB_OBJS) $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(LDLIBS) - $(Q) ln -sf $(SHARED_PREFIX)upb_textformat_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libupb_textformat_lib$(SHARED_VERSION_CORE).so.38 + $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libupb_textformat_lib.so.39 -o $(LIBDIR)/$(CONFIG)/libupb_textformat_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUPB_TEXTFORMAT_LIB_OBJS) $(LIBDIR)/$(CONFIG)/libutf8_range_lib.a $(LIBDIR)/$(CONFIG)/libupb_message_lib.a $(LIBDIR)/$(CONFIG)/libupb_mem_lib.a $(LIBDIR)/$(CONFIG)/libupb_base_lib.a $(LDLIBS) + $(Q) ln -sf $(SHARED_PREFIX)upb_textformat_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libupb_textformat_lib$(SHARED_VERSION_CORE).so.39 $(Q) ln -sf $(SHARED_PREFIX)upb_textformat_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libupb_textformat_lib$(SHARED_VERSION_CORE).so endif endif @@ -2820,8 +2820,8 @@ $(LIBDIR)/$(CONFIG)/libutf8_range_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): ifeq ($(SYSTEM),Darwin) $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)utf8_range_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libutf8_range_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUTF8_RANGE_LIB_OBJS) $(LDLIBS) else - $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libutf8_range_lib.so.38 -o $(LIBDIR)/$(CONFIG)/libutf8_range_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUTF8_RANGE_LIB_OBJS) $(LDLIBS) - $(Q) ln -sf $(SHARED_PREFIX)utf8_range_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libutf8_range_lib$(SHARED_VERSION_CORE).so.38 + $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libutf8_range_lib.so.39 -o $(LIBDIR)/$(CONFIG)/libutf8_range_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBUTF8_RANGE_LIB_OBJS) $(LDLIBS) + $(Q) ln -sf $(SHARED_PREFIX)utf8_range_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libutf8_range_lib$(SHARED_VERSION_CORE).so.39 $(Q) ln -sf $(SHARED_PREFIX)utf8_range_lib$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libutf8_range_lib$(SHARED_VERSION_CORE).so endif endif diff --git a/build_config.rb b/build_config.rb index 90c78c31080..c8e590fdd15 100644 --- a/build_config.rb +++ b/build_config.rb @@ -13,5 +13,5 @@ # limitations under the License. module GrpcBuildConfig - CORE_WINDOWS_DLL = '/tmp/libs/opt/grpc-38.dll' + CORE_WINDOWS_DLL = '/tmp/libs/opt/grpc-39.dll' end diff --git a/build_handwritten.yaml b/build_handwritten.yaml index c40860bfc54..ced47463e95 100644 --- a/build_handwritten.yaml +++ b/build_handwritten.yaml @@ -12,7 +12,7 @@ settings: '#08': Use "-preN" suffixes to identify pre-release versions '#09': Per-language overrides are possible with (eg) ruby_version tag here '#10': See the expand_version.py for all the quirks here - core_version: 38.0.0 + core_version: 39.0.0 csharp_major_version: 2 g_stands_for: guardian protobuf_version: 3.25.1 diff --git a/src/core/lib/surface/version.cc b/src/core/lib/surface/version.cc index 21ec5ed5b5a..f27659c6e57 100644 --- a/src/core/lib/surface/version.cc +++ b/src/core/lib/surface/version.cc @@ -23,6 +23,6 @@ #include -const char* grpc_version_string(void) { return "38.0.0"; } +const char* grpc_version_string(void) { return "39.0.0"; } const char* grpc_g_stands_for(void) { return "guardian"; } diff --git a/src/objective-c/tests/version.h b/src/objective-c/tests/version.h index 5b45207a920..135cb772c77 100644 --- a/src/objective-c/tests/version.h +++ b/src/objective-c/tests/version.h @@ -23,4 +23,4 @@ // `tools/buildgen/generate_projects.sh`. #define GRPC_OBJC_VERSION_STRING @"1.62.0-dev" -#define GRPC_C_VERSION_STRING @"38.0.0" +#define GRPC_C_VERSION_STRING @"39.0.0" diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index 50885cee19f..98e7f3f40c5 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC Core" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 38.0.0 +PROJECT_NUMBER = 39.0.0 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 732cae90a6e..9ee3aeff6a1 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC Core" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 38.0.0 +PROJECT_NUMBER = 39.0.0 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a From 1a198795fa63dd9f861622b0aad3fc08ce6d5d46 Mon Sep 17 00:00:00 2001 From: AJ Heller Date: Tue, 13 Feb 2024 11:02:55 -0800 Subject: [PATCH 06/37] [experiment] Disable chaotic_good tests on Windows (#35897) Experiment test failures are blocking fixes on Windows. Closes #35897 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/35897 from drfloob:no-chaotic-good-on-windows 5190d16e608c8388f15efed407b89ad8432d33b4 PiperOrigin-RevId: 606681739 --- bazel/experiments.bzl | 2 -- src/core/lib/experiments/rollouts.yaml | 5 +++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/bazel/experiments.bzl b/bazel/experiments.bzl index 3b8b6ddf3ff..4712d5802e7 100644 --- a/bazel/experiments.bzl +++ b/bazel/experiments.bzl @@ -77,7 +77,6 @@ EXPERIMENTS = { "v3_compression_filter", ], "core_end2end_test": [ - "chaotic_good", "promise_based_client_call", "promise_based_server_call", "work_serializer_dispatch", @@ -157,7 +156,6 @@ EXPERIMENTS = { "v3_compression_filter", ], "core_end2end_test": [ - "chaotic_good", "promise_based_client_call", "promise_based_server_call", "work_serializer_dispatch", diff --git a/src/core/lib/experiments/rollouts.yaml b/src/core/lib/experiments/rollouts.yaml index 26e77b8f800..0f89e8ece39 100644 --- a/src/core/lib/experiments/rollouts.yaml +++ b/src/core/lib/experiments/rollouts.yaml @@ -46,6 +46,11 @@ default: false - name: canary_client_privacy default: false +- name: chaotic_good + default: + ios: broken + posix: false + windows: broken - name: client_idleness default: true - name: client_privacy From 385431ca4b41748ab11d4ba07645d818b725ab84 Mon Sep 17 00:00:00 2001 From: Shane Loretz Date: Tue, 13 Feb 2024 11:42:40 -0800 Subject: [PATCH 07/37] [Python Bazel] Allow overwriting grpc_library in python_rules.bzl (#35629) Continues #35412 addressing feedback in https://github.com/grpc/grpc/pull/35412#issuecomment-1875980755 I'm unable to test the workspace with the newly added py_test, so I'm hoping a CI run here will tell me if the test works. ```console ~/grpc/test/distrib/bazel/python$ bazel --nohome_rc --nosystem_rc test //... Loading: Loading: Loading: 0 packages loaded Analyzing: 37 targets (0 packages loaded, 0 targets configured) ERROR: /usr/local/foobar/home/sloretz/.cache/bazel/_bazel_sloretz/7f83b4f00f370e7c52a5cc586445673c/external/com_google_protobuf/upb_generator/BUILD:266:21: @com_google_protobuf//upb_generator:protoc-gen-upb_toolchain: no such attribute 'output_files' in 'proto_lang_toolchain' rule ERROR: /usr/local/foobar/home/sloretz/.cache/bazel/_bazel_sloretz/7f83b4f00f370e7c52a5cc586445673c/external/com_google_protobuf/upb_generator/BUILD:305:21: @com_google_protobuf//upb_generator:protoc-gen-upb_minitable_toolchain: no such attribute 'output_files' in 'proto_lang_toolchain' rule ERROR: /usr/local/foobar/home/sloretz/.cache/bazel/_bazel_sloretz/7f83b4f00f370e7c52a5cc586445673c/external/com_google_protobuf/upb_generator/BUILD:338:21: @com_google_protobuf//upb_generator:protoc-gen-upbdefs_toolchain: no such attribute 'output_files' in 'proto_lang_toolchain' rule ERROR: /usr/local/foobar/home/sloretz/.cache/bazel/_bazel_sloretz/7f83b4f00f370e7c52a5cc586445673c/external/com_google_protobuf/upb_generator/BUILD:305:21: Target '@com_google_protobuf//upb_generator:protoc-gen-upb_minitable_stage1' contains an error and its package is in error and referenced by '@com_google_protobuf//upb_generator:protoc-gen-upb_minitable_toolchain' ERROR: /usr/local/foobar/home/sloretz/.cache/bazel/_bazel_sloretz/7f83b4f00f370e7c52a5cc586445673c/external/com_github_grpc_grpc/src/proto/grpc/gcp/BUILD:19:14: every rule of type proto_library implicitly depends upon the target '@com_google_protobuf//upb_generator:protoc-gen-upb_minitable_toolchain', but this target could not be found because of: Target '@com_google_protobuf//upb_generator:protoc-gen-upb_minitable_toolchain' contains an error and its package is in error ERROR: Analysis failed ERROR: Analysis of target '//namespaced/upper/example:no_import_no_strip_py_pb2_grpc' failed; build aborted: INFO: Elapsed time: 0.171s INFO: 0 processes. FAILED: Build did NOT complete successfully (0 packages loaded, 0 targets configured) ERROR: Couldn't start the build. Unable to run tests ``` Closes #35629 PiperOrigin-RevId: 606695865 --- bazel/python_rules.bzl | 11 +++++--- test/distrib/bazel/python/BUILD | 25 +++++++++++++++++++ .../bazel/python/grpc_library_replacement.py | 16 ++++++++++++ .../python/grpc_library_replacement_test.py | 24 ++++++++++++++++++ 4 files changed, 73 insertions(+), 3 deletions(-) create mode 100644 test/distrib/bazel/python/grpc_library_replacement.py create mode 100644 test/distrib/bazel/python/grpc_library_replacement_test.py diff --git a/bazel/python_rules.bzl b/bazel/python_rules.bzl index 0bddf921ce9..9c19d7ac2f2 100644 --- a/bazel/python_rules.bzl +++ b/bazel/python_rules.bzl @@ -222,11 +222,11 @@ def _generate_pb2_grpc_src_impl(context): py_info = _merge_pyinfos( [ p, - context.attr._grpc_library[PyInfo], + context.attr.grpc_library[PyInfo], ] + [dep[PyInfo] for dep in context.attr.py_deps], ) - runfiles = context.runfiles(files = out_files, transitive_files = py_info.transitive_sources).merge(context.attr._grpc_library[DefaultInfo].data_runfiles) + runfiles = context.runfiles(files = out_files, transitive_files = py_info.transitive_sources).merge(context.attr.grpc_library[DefaultInfo].data_runfiles) return [ DefaultInfo( @@ -261,7 +261,7 @@ _generate_pb2_grpc_src = rule( cfg = "exec", default = Label("//external:protocol_compiler"), ), - "_grpc_library": attr.label( + "grpc_library": attr.label( default = Label("//src/python/grpcio/grpc:grpcio"), providers = [PyInfo], ), @@ -274,6 +274,7 @@ def py_grpc_library( srcs, deps, strip_prefixes = [], + grpc_library = Label("//src/python/grpcio/grpc:grpcio"), **kwargs): """Generate python code for gRPC services defined in a protobuf. @@ -287,6 +288,9 @@ def py_grpc_library( stripped from the beginning of foo_pb2 modules imported by the generated stubs. This is useful in combination with the `imports` attribute of the `py_library` rule. + grpc_library: (`label`) a single `py_library` target representing the + python gRPC library target to be depended upon. This can be used to + generate code that depends on `grpcio` from the Python Package Index. **kwargs: Additional arguments to be supplied to the invocation of py_library. """ @@ -301,5 +305,6 @@ def py_grpc_library( deps = srcs, py_deps = deps, strip_prefixes = strip_prefixes, + grpc_library = grpc_library, **kwargs ) diff --git a/test/distrib/bazel/python/BUILD b/test/distrib/bazel/python/BUILD index f784dd8796f..39634b0e633 100644 --- a/test/distrib/bazel/python/BUILD +++ b/test/distrib/bazel/python/BUILD @@ -170,3 +170,28 @@ py_test( python_rules_test_suite( name = "python_rules_test", ) + +# Test that grpc_library attribute replaces grpcio dependency on +# py_grpc_library targets + +py_library( + name = "grpc_library_replacement", + srcs = ["grpc_library_replacement.py"], +) + +py_grpc_library( + name = "helloworld_py_pb2_grpc_library_changed", + srcs = [":helloworld_proto"], + grpc_library = ":grpc_library_replacement", + deps = [":helloworld_py_pb2"], +) + +py_test( + name = "grpc_library_replacement_test", + srcs = ["grpc_library_replacement_test.py"], + main = "grpc_library_replacement_test.py", + python_version = "PY3", + deps = [ + ":helloworld_py_pb2_grpc_library_changed", + ], +) diff --git a/test/distrib/bazel/python/grpc_library_replacement.py b/test/distrib/bazel/python/grpc_library_replacement.py new file mode 100644 index 00000000000..334a30ef42f --- /dev/null +++ b/test/distrib/bazel/python/grpc_library_replacement.py @@ -0,0 +1,16 @@ +# Copyright 2024 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Intentionally empty. +# This is used to test the grpc_library attribute of py_grpc_library works. diff --git a/test/distrib/bazel/python/grpc_library_replacement_test.py b/test/distrib/bazel/python/grpc_library_replacement_test.py new file mode 100644 index 00000000000..fb3e449c07f --- /dev/null +++ b/test/distrib/bazel/python/grpc_library_replacement_test.py @@ -0,0 +1,24 @@ +# Copyright 2024 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +try: + import grpc +except ImportError: + pass +else: + sys.exit("Unexpectedly able to import grpc") + +import grpc_library_replacement From 98a96c5068da14ed29d70ca23818b5f408a2e7b4 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Tue, 13 Feb 2024 11:53:56 -0800 Subject: [PATCH 08/37] [Protobuf] Absorb protobuf::MultiFileErrorCollector change 2 (#35898) Additional touch on top of https://github.com/grpc/grpc/pull/35870 Closes #35898 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/35898 from veblush:protobuf-sv 3b8410b6f6e574c9069196acd558184230e9196e PiperOrigin-RevId: 606699149 --- test/cpp/util/proto_file_parser.cc | 8 ++++---- tools/distrib/python/grpcio_tools/grpc_tools/main.cc | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/cpp/util/proto_file_parser.cc b/test/cpp/util/proto_file_parser.cc index 97ac68f1795..5e570c57372 100644 --- a/test/cpp/util/proto_file_parser.cc +++ b/test/cpp/util/proto_file_parser.cc @@ -48,16 +48,16 @@ class ErrorPrinter : public protobuf::compiler::MultiFileErrorCollector { public: explicit ErrorPrinter(ProtoFileParser* parser) : parser_(parser) {} - void AddError(const std::string& filename, int line, int column, - const std::string& message) override { + void RecordError(absl::string_view filename, int line, int column, + absl::string_view message) override { std::ostringstream oss; oss << "error " << filename << " " << line << " " << column << " " << message << "\n"; parser_->LogError(oss.str()); } - void AddWarning(const std::string& filename, int line, int column, - const std::string& message) override { + void RecordWarning(absl::string_view filename, int line, int column, + absl::string_view message) override { std::cerr << "warning " << filename << " " << line << " " << column << " " << message << std::endl; } diff --git a/tools/distrib/python/grpcio_tools/grpc_tools/main.cc b/tools/distrib/python/grpcio_tools/grpc_tools/main.cc index 1d9749861bc..9302cc75529 100644 --- a/tools/distrib/python/grpcio_tools/grpc_tools/main.cc +++ b/tools/distrib/python/grpcio_tools/grpc_tools/main.cc @@ -108,13 +108,13 @@ class ErrorCollectorImpl : public MultiFileErrorCollector { : errors_(errors), warnings_(warnings) {} void RecordError(absl::string_view filename, int line, int column, - absl::string_view message) { + absl::string_view message) override { errors_->emplace_back(std::string(filename), line, column, std::string(message)); } void RecordWarning(absl::string_view filename, int line, int column, - absl::string_view message) { + absl::string_view message) override { warnings_->emplace_back(std::string(filename), line, column, std::string(message)); } From 46a261e96a28634c19a97ba6c02088ac2e4b2ef5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 13 Feb 2024 12:21:40 -0800 Subject: [PATCH 09/37] [chaotic-good] Fix write ordering (#35900) It could happen that we start two writes on the same endpoint simultaneously, leading to awesome results. Closes #35900 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/35900 from ctiller:write-order 1f7559d7b97dfd753de007735571e44a4551e959 PiperOrigin-RevId: 606707912 --- src/core/BUILD | 1 + .../client/chaotic_good_connector.cc | 36 ++++++-------- .../server/chaotic_good_server.cc | 46 ++++++++--------- src/core/lib/transport/promise_endpoint.h | 49 ++++++++++++++----- 4 files changed, 75 insertions(+), 57 deletions(-) diff --git a/src/core/BUILD b/src/core/BUILD index e965464e8fc..b4174ba9988 100644 --- a/src/core/BUILD +++ b/src/core/BUILD @@ -6562,6 +6562,7 @@ grpc_cc_library( ], deps = [ "activity", + "cancel_callback", "event_engine_common", "if", "map", diff --git a/src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc b/src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc index b6f80e6de46..6be26f47c43 100644 --- a/src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc +++ b/src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc @@ -105,16 +105,14 @@ auto ChaoticGoodConnector::DataEndpointReadSettingsFrame( auto ChaoticGoodConnector::DataEndpointWriteSettingsFrame( RefCountedPtr self) { - return [self]() { - // Serialize setting frame. - SettingsFrame frame; - // frame.header set connectiion_type: control - frame.headers = SettingsMetadata{SettingsMetadata::ConnectionType::kData, - self->connection_id_, kDataAlignmentBytes} - .ToMetadataBatch(GetContext()); - auto write_buffer = frame.Serialize(&self->hpack_compressor_); - return self->data_endpoint_.Write(std::move(write_buffer.control)); - }; + // Serialize setting frame. + SettingsFrame frame; + // frame.header set connectiion_type: control + frame.headers = SettingsMetadata{SettingsMetadata::ConnectionType::kData, + self->connection_id_, kDataAlignmentBytes} + .ToMetadataBatch(GetContext()); + auto write_buffer = frame.Serialize(&self->hpack_compressor_); + return self->data_endpoint_.Write(std::move(write_buffer.control)); } auto ChaoticGoodConnector::WaitForDataEndpointSetup( @@ -200,16 +198,14 @@ auto ChaoticGoodConnector::ControlEndpointReadSettingsFrame( auto ChaoticGoodConnector::ControlEndpointWriteSettingsFrame( RefCountedPtr self) { - return [self]() { - // Serialize setting frame. - SettingsFrame frame; - // frame.header set connectiion_type: control - frame.headers = SettingsMetadata{SettingsMetadata::ConnectionType::kControl, - absl::nullopt, absl::nullopt} - .ToMetadataBatch(GetContext()); - auto write_buffer = frame.Serialize(&self->hpack_compressor_); - return self->control_endpoint_.Write(std::move(write_buffer.control)); - }; + // Serialize setting frame. + SettingsFrame frame; + // frame.header set connectiion_type: control + frame.headers = SettingsMetadata{SettingsMetadata::ConnectionType::kControl, + absl::nullopt, absl::nullopt} + .ToMetadataBatch(GetContext()); + auto write_buffer = frame.Serialize(&self->hpack_compressor_); + return self->control_endpoint_.Write(std::move(write_buffer.control)); } void ChaoticGoodConnector::Connect(const Args& args, Result* result, diff --git a/src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc b/src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc index 472e166bf5f..4359d5648ad 100644 --- a/src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc +++ b/src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc @@ -331,37 +331,29 @@ auto ChaoticGoodServerListener::ActiveConnection::HandshakingState:: auto ChaoticGoodServerListener::ActiveConnection::HandshakingState:: ControlEndpointWriteSettingsFrame(RefCountedPtr self) { + self->connection_->NewConnectionID(); + SettingsFrame frame; + frame.headers = + SettingsMetadata{absl::nullopt, self->connection_->connection_id_, + absl::nullopt} + .ToMetadataBatch(GetContext()); + auto write_buffer = frame.Serialize(&self->connection_->hpack_compressor_); return TrySeq( - [self]() { - self->connection_->NewConnectionID(); - SettingsFrame frame; - frame.headers = - SettingsMetadata{absl::nullopt, self->connection_->connection_id_, - absl::nullopt} - .ToMetadataBatch(GetContext()); - auto write_buffer = - frame.Serialize(&self->connection_->hpack_compressor_); - return self->connection_->endpoint_.Write( - std::move(write_buffer.control)); - }, + self->connection_->endpoint_.Write(std::move(write_buffer.control)), WaitForDataEndpointSetup(self)); } auto ChaoticGoodServerListener::ActiveConnection::HandshakingState:: DataEndpointWriteSettingsFrame(RefCountedPtr self) { + // Send data endpoint setting frame + SettingsFrame frame; + frame.headers = + SettingsMetadata{absl::nullopt, self->connection_->connection_id_, + self->connection_->data_alignment_} + .ToMetadataBatch(GetContext()); + auto write_buffer = frame.Serialize(&self->connection_->hpack_compressor_); return TrySeq( - [self]() { - // Send data endpoint setting frame - SettingsFrame frame; - frame.headers = - SettingsMetadata{absl::nullopt, self->connection_->connection_id_, - self->connection_->data_alignment_} - .ToMetadataBatch(GetContext()); - auto write_buffer = - frame.Serialize(&self->connection_->hpack_compressor_); - return self->connection_->endpoint_.Write( - std::move(write_buffer.control)); - }, + self->connection_->endpoint_.Write(std::move(write_buffer.control)), [self]() mutable { MutexLock lock(&self->connection_->listener_->mu_); // Set endpoint to latch @@ -380,8 +372,10 @@ auto ChaoticGoodServerListener::ActiveConnection::HandshakingState:: auto ChaoticGoodServerListener::ActiveConnection::HandshakingState:: EndpointWriteSettingsFrame(RefCountedPtr self, bool is_control_endpoint) { - return If(is_control_endpoint, ControlEndpointWriteSettingsFrame(self), - DataEndpointWriteSettingsFrame(self)); + return If( + is_control_endpoint, + [&self] { return ControlEndpointWriteSettingsFrame(self); }, + [&self] { return DataEndpointWriteSettingsFrame(self); }); } void ChaoticGoodServerListener::ActiveConnection::HandshakingState:: diff --git a/src/core/lib/transport/promise_endpoint.h b/src/core/lib/transport/promise_endpoint.h index 12cd0969284..bcbf13faf8e 100644 --- a/src/core/lib/transport/promise_endpoint.h +++ b/src/core/lib/transport/promise_endpoint.h @@ -39,6 +39,7 @@ #include "src/core/lib/gprpp/sync.h" #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/promise/activity.h" +#include "src/core/lib/promise/cancel_callback.h" #include "src/core/lib/promise/if.h" #include "src/core/lib/promise/map.h" #include "src/core/lib/promise/poll.h" @@ -69,8 +70,10 @@ class PromiseEndpoint { // `Write()` before the previous write finishes. Doing that results in // undefined behavior. auto Write(SliceBuffer data) { - // Assert previous write finishes. - GPR_ASSERT(!write_state_->complete.load(std::memory_order_relaxed)); + // Start write and assert previous write finishes. + auto prev = write_state_->state.exchange(WriteState::kWriting, + std::memory_order_relaxed); + GPR_ASSERT(prev == WriteState::kIdle); bool completed; if (data.Length() == 0) { completed = true; @@ -92,16 +95,31 @@ class PromiseEndpoint { if (completed) write_state_->waker = Waker(); } return If( - completed, []() { return []() { return absl::OkStatus(); }; }, + completed, + [this]() { + return [write_state = write_state_]() { + auto prev = write_state->state.exchange(WriteState::kIdle, + std::memory_order_relaxed); + GPR_ASSERT(prev == WriteState::kWriting); + return absl::OkStatus(); + }; + }, [this]() { return [write_state = write_state_]() -> Poll { - // If current write isn't finished return `Pending()`, else return - // write result. - if (!write_state->complete.load(std::memory_order_acquire)) { - return Pending(); + // If current write isn't finished return `Pending()`, else + // return write result. + WriteState::State expected = WriteState::kWritten; + if (write_state->state.compare_exchange_strong( + expected, WriteState::kIdle, std::memory_order_acquire, + std::memory_order_relaxed)) { + // State was Written, and we changed it to Idle. We can return + // the result. + return std::move(write_state->result); } - write_state->complete.store(false, std::memory_order_relaxed); - return std::move(write_state->result); + // State was not Written; since we're polling it must be + // Writing. Assert that and return Pending. + GPR_ASSERT(expected == WriteState::kWriting); + return Pending(); }; }); } @@ -228,7 +246,13 @@ class PromiseEndpoint { }; struct WriteState : public RefCounted { - std::atomic complete{false}; + enum State : uint8_t { + kIdle, // Not writing. + kWriting, // Write started, but not completed. + kWritten, // Write completed. + }; + + std::atomic state{kIdle}; // Write buffer used for `EventEngine::Endpoint::Write()` to ensure the // memory behind the buffer is not lost. grpc_event_engine::experimental::SliceBuffer buffer; @@ -239,7 +263,10 @@ class PromiseEndpoint { void Complete(absl::Status status) { result = std::move(status); auto w = std::move(waker); - complete.store(true, std::memory_order_release); + auto prev = state.exchange(kWritten, std::memory_order_release); + // Previous state should be Writing. If we got anything else we've entered + // the callback path twice. + GPR_ASSERT(prev == kWriting); w.Wakeup(); } }; From 1669b9e8538505808280df55f3287cf0fd2d234e Mon Sep 17 00:00:00 2001 From: apolcyn Date: Tue, 13 Feb 2024 12:22:05 -0800 Subject: [PATCH 10/37] [testing] Add ExitIdle and CompleteCall helpers to LB policy test lib (#35860) As title. Pulling these additions out from a larger change. Related: cl/563857636 Closes #35860 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/35860 from apolcyn:test_lib_changes 40b6455638d0e7bbe1c2d8b15b38f09c9e7c1ce5 PiperOrigin-RevId: 606708025 --- .../lb_policy/lb_policy_test_lib.h | 41 +++++++++++++++---- .../lb_policy/pick_first_test.cc | 14 +------ .../lb_policy/xds_override_host_test.cc | 11 ++--- 3 files changed, 41 insertions(+), 25 deletions(-) diff --git a/test/core/client_channel/lb_policy/lb_policy_test_lib.h b/test/core/client_channel/lb_policy/lb_policy_test_lib.h index 3c8b8c8de31..3b75537461e 100644 --- a/test/core/client_channel/lb_policy/lb_policy_test_lib.h +++ b/test/core/client_channel/lb_policy/lb_policy_test_lib.h @@ -861,6 +861,23 @@ class LoadBalancingPolicyTest : public ::testing::Test { return status; } + // Invoke ExitIdle on the LB policy + void ExitIdle() { + ExecCtx exec_ctx; + absl::Notification notification; + // Note: ExitIdle() will enqueue a bunch of connectivity state + // notifications on the WorkSerializer, and we want to wait until + // those are delivered to the LB policy. + work_serializer_->Run( + [&]() { + lb_policy_->ExitIdleLocked(); + work_serializer_->Run([&]() { notification.Notify(); }, + DEBUG_LOCATION); + }, + DEBUG_LOCATION); + notification.WaitForNotification(); + } + void ExpectQueueEmpty(SourceLocation location = SourceLocation()) { helper_->ExpectQueueEmpty(location); } @@ -1087,6 +1104,7 @@ class LoadBalancingPolicyTest : public ::testing::Test { const CallAttributes& call_attributes = {}, std::unique_ptr* subchannel_call_tracker = nullptr, + SubchannelState::FakeSubchannel** picked_subchannel = nullptr, SourceLocation location = SourceLocation()) { EXPECT_NE(picker, nullptr); if (picker == nullptr) { @@ -1100,22 +1118,31 @@ class LoadBalancingPolicyTest : public ::testing::Test { if (complete == nullptr) return absl::nullopt; auto* subchannel = static_cast( complete->subchannel.get()); + if (picked_subchannel != nullptr) *picked_subchannel = subchannel; std::string address = subchannel->state()->address(); if (complete->subchannel_call_tracker != nullptr) { if (subchannel_call_tracker != nullptr) { *subchannel_call_tracker = std::move(complete->subchannel_call_tracker); } else { - complete->subchannel_call_tracker->Start(); - FakeMetadata metadata({}); - FakeBackendMetricAccessor backend_metric_accessor({}); - LoadBalancingPolicy::SubchannelCallTrackerInterface::FinishArgs args = { - address, absl::OkStatus(), &metadata, &backend_metric_accessor}; - complete->subchannel_call_tracker->Finish(args); + ReportCompletionToCallTracker( + std::move(complete->subchannel_call_tracker), address); } } return address; } + void ReportCompletionToCallTracker( + std::unique_ptr + subchannel_call_tracker, + absl::string_view address, absl::Status status = absl::OkStatus()) { + subchannel_call_tracker->Start(); + FakeMetadata metadata({}); + FakeBackendMetricAccessor backend_metric_accessor({}); + LoadBalancingPolicy::SubchannelCallTrackerInterface::FinishArgs args = { + address, status, &metadata, &backend_metric_accessor}; + subchannel_call_tracker->Finish(args); + } + // Gets num_picks complete picks from picker and returns the resulting // list of addresses, or nullopt if a non-complete pick was returned. absl::optional> GetCompletePicks( @@ -1137,7 +1164,7 @@ class LoadBalancingPolicyTest : public ::testing::Test { subchannel_call_trackers == nullptr ? nullptr : &subchannel_call_tracker, - location); + nullptr, location); if (!address.has_value()) return absl::nullopt; results.emplace_back(std::move(*address)); if (subchannel_call_trackers != nullptr) { diff --git a/test/core/client_channel/lb_policy/pick_first_test.cc b/test/core/client_channel/lb_policy/pick_first_test.cc index b5cdf28801e..fccc8af16ee 100644 --- a/test/core/client_channel/lb_policy/pick_first_test.cc +++ b/test/core/client_channel/lb_policy/pick_first_test.cc @@ -76,20 +76,8 @@ class PickFirstTest : public LoadBalancingPolicyTest { void GetOrderAddressesArePicked( absl::Span addresses, std::vector* out_address_order) { - ExecCtx exec_ctx; out_address_order->clear(); - // Note: ExitIdle() will enqueue a bunch of connectivity state - // notifications on the WorkSerializer, and we want to wait until - // those are delivered to the LB policy. - absl::Notification notification; - work_serializer_->Run( - [&]() { - lb_policy()->ExitIdleLocked(); - work_serializer_->Run([&]() { notification.Notify(); }, - DEBUG_LOCATION); - }, - DEBUG_LOCATION); - notification.WaitForNotification(); + ExitIdle(); // Construct a map of subchannel to address. // We will remove entries as each subchannel starts to connect. std::map subchannels; diff --git a/test/core/client_channel/lb_policy/xds_override_host_test.cc b/test/core/client_channel/lb_policy/xds_override_host_test.cc index de6ec9fb9bd..d1ddeae05c7 100644 --- a/test/core/client_channel/lb_policy/xds_override_host_test.cc +++ b/test/core/client_channel/lb_policy/xds_override_host_test.cc @@ -187,10 +187,10 @@ class XdsOverrideHostTest : public LoadBalancingPolicyTest { } std::string expected_addresses_str = absl::StrJoin(expected_addresses, ","); for (size_t i = 0; i < 3; ++i) { - EXPECT_EQ( - ExpectPickComplete(picker, {attribute}, - /*subchannel_call_tracker=*/nullptr, location), - expected) + EXPECT_EQ(ExpectPickComplete(picker, {attribute}, + /*subchannel_call_tracker=*/nullptr, + /*picked_subchannel=*/nullptr, location), + expected) << location.file() << ":" << location.line(); EXPECT_EQ(attribute->actual_address_list(), expected_addresses_str) << " Actual: " << attribute->actual_address_list() << "\n" @@ -207,7 +207,8 @@ class XdsOverrideHostTest : public LoadBalancingPolicyTest { std::vector actual_picks; for (size_t i = 0; i < expected.size(); ++i) { auto address = ExpectPickComplete( - picker, {attribute}, /*subchannel_call_tracker=*/nullptr, location); + picker, {attribute}, /*subchannel_call_tracker=*/nullptr, + /*picked_subchannel=*/nullptr, location); ASSERT_TRUE(address.has_value()) << location.file() << ":" << location.line(); EXPECT_THAT(*address, ::testing::AnyOfArray(expected)) From 510dbc92e1e0999f282d31ca80927d45059a02a0 Mon Sep 17 00:00:00 2001 From: AJ Heller Date: Tue, 13 Feb 2024 14:42:23 -0800 Subject: [PATCH 11/37] [EventEngine] Fix lock acquisition ordering problem on Windows Connect (#35892) CC @youyuanwu This fixes a lock acquisition ordering problem in WindowsEventEngine's Connect behavior, between a ConnectionState's internal mutex, and a HandshakeManager's mutex. The connection state mutex should not be held when calling the on_connected callback. This should unblock #34801 Closes #35892 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/35892 from drfloob:connect-callback-lock-cycle 4d56120e2f6583ff52b068350c39e514126b4543 PiperOrigin-RevId: 606752276 --- .../event_engine/windows/windows_engine.cc | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/src/core/lib/event_engine/windows/windows_engine.cc b/src/core/lib/event_engine/windows/windows_engine.cc index 69e4963f387..5503f3fc85b 100644 --- a/src/core/lib/event_engine/windows/windows_engine.cc +++ b/src/core/lib/event_engine/windows/windows_engine.cc @@ -49,6 +49,13 @@ namespace grpc_event_engine { namespace experimental { +namespace { +EventEngine::OnConnectCallback CreateCrashingOnConnectCallback() { + return [](absl::StatusOr>) { + grpc_core::Crash("Internal Error: OnConnect callback called when unset"); + }; +} +} // namespace // ---- IOCPWorkClosure ---- WindowsEventEngine::IOCPWorkClosure::IOCPWorkClosure(ThreadPool* thread_pool, @@ -250,6 +257,7 @@ void WindowsEventEngine::OnConnectCompleted( // Connection attempt complete! grpc_core::MutexLock lock(&state->mu); cb = std::move(state->on_connected_user_callback); + state->on_connected_user_callback = CreateCrashingOnConnectCallback(); state->on_connected = nullptr; { grpc_core::MutexLock handle_lock(&connection_mu_); @@ -356,10 +364,13 @@ EventEngine::ConnectionHandle WindowsEventEngine::Connect( }); connection_state->timer_handle = RunAfter(timeout, [this, connection_state]() { - grpc_core::MutexLock lock(&connection_state->mu); + grpc_core::ReleasableMutexLock lock(&connection_state->mu); if (CancelConnectFromDeadlineTimer(connection_state.get())) { - connection_state->on_connected_user_callback( - absl::DeadlineExceededError("Connection timed out")); + auto cb = std::move(connection_state->on_connected_user_callback); + connection_state->on_connected_user_callback = + CreateCrashingOnConnectCallback(); + lock.Release(); + cb(absl::DeadlineExceededError("Connection timed out")); } // else: The connection attempt could not be canceled. We can assume // the connection callback will be called. @@ -380,8 +391,14 @@ EventEngine::ConnectionHandle WindowsEventEngine::Connect( connection_state->socket->Shutdown(DEBUG_LOCATION, "ConnectEx"); Run([connection_state = std::move(connection_state), status = GRPC_WSA_ERROR(WSAGetLastError(), "ConnectEx")]() mutable { - grpc_core::MutexLock lock(&connection_state->mu); - connection_state->on_connected_user_callback(status); + EventEngine::OnConnectCallback cb; + { + grpc_core::MutexLock lock(&connection_state->mu); + cb = std::move(connection_state->on_connected_user_callback); + connection_state->on_connected_user_callback = + CreateCrashingOnConnectCallback(); + } + cb(status); }); return EventEngine::ConnectionHandle::kInvalid; } From 17d9e20ff1293139b0ccca5c5d5ce8df83715a90 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Tue, 13 Feb 2024 15:03:03 -0800 Subject: [PATCH 12/37] [slice] use absl base64 APIs instead of slice base64 APIs (#35851) Closes #35851 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/35851 from markdroth:base64 6caf83d3dad4f6448b2b3cbb2ace36ab535ae602 PiperOrigin-RevId: 606759013 --- BUILD | 1 + bazel/experiments.bzl | 10 ++++ src/core/client_channel/http_proxy_mapper.cc | 14 ++++- src/core/lib/experiments/experiments.cc | 12 ++++ src/core/lib/experiments/experiments.h | 11 ++++ src/core/lib/experiments/experiments.yaml | 5 ++ src/core/lib/experiments/rollouts.yaml | 2 + .../security/credentials/jwt/json_token.cc | 22 ++++++- .../security/credentials/jwt/jwt_verifier.cc | 58 +++++++++++++------ .../end2end/fixtures/http_proxy_fixture.cc | 20 +++---- test/core/security/BUILD | 2 + test/core/security/json_token_test.cc | 40 ++++++------- test/core/security/jwt_verifier_test.cc | 24 +++----- test/cpp/interop/client_helper.cc | 11 ++-- 14 files changed, 152 insertions(+), 80 deletions(-) diff --git a/BUILD b/BUILD index 455b7db1705..747cf55f4c1 100644 --- a/BUILD +++ b/BUILD @@ -3384,6 +3384,7 @@ grpc_cc_library( "//src/core:arena_promise", "//src/core:closure", "//src/core:error", + "//src/core:experiments", "//src/core:gpr_manual_constructor", "//src/core:httpcli_ssl_credentials", "//src/core:iomgr_fwd", diff --git a/bazel/experiments.bzl b/bazel/experiments.bzl index 4712d5802e7..5281d7d5b07 100644 --- a/bazel/experiments.bzl +++ b/bazel/experiments.bzl @@ -17,6 +17,7 @@ """Dictionary of tags to experiments so we know when to test different experiments.""" EXPERIMENT_ENABLES = { + "absl_base64": "absl_base64", "call_status_override_on_cancellation": "call_status_override_on_cancellation", "call_v3": "call_v3", "canary_client_privacy": "canary_client_privacy", @@ -123,6 +124,9 @@ EXPERIMENTS = { "round_robin_delegate_to_pick_first", "wrr_delegate_to_pick_first", ], + "credential_token_tests": [ + "absl_base64", + ], "event_engine_listener_test": [ "event_engine_listener", ], @@ -199,6 +203,9 @@ EXPERIMENTS = { "round_robin_delegate_to_pick_first", "wrr_delegate_to_pick_first", ], + "credential_token_tests": [ + "absl_base64", + ], "flow_control_test": [ "write_size_cap", "write_size_policy", @@ -286,6 +293,9 @@ EXPERIMENTS = { "round_robin_delegate_to_pick_first", "wrr_delegate_to_pick_first", ], + "credential_token_tests": [ + "absl_base64", + ], "event_engine_listener_test": [ "event_engine_listener", ], diff --git a/src/core/client_channel/http_proxy_mapper.cc b/src/core/client_channel/http_proxy_mapper.cc index da59b76c10e..a586a75945d 100644 --- a/src/core/client_channel/http_proxy_mapper.cc +++ b/src/core/client_channel/http_proxy_mapper.cc @@ -30,6 +30,7 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/ascii.h" +#include "absl/strings/escaping.h" #include "absl/strings/match.h" #include "absl/strings/numbers.h" #include "absl/strings/str_cat.h" @@ -45,6 +46,7 @@ #include "src/core/lib/address_utils/parse_address.h" #include "src/core/lib/address_utils/sockaddr_utils.h" #include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/experiments/experiments.h" #include "src/core/lib/gpr/string.h" #include "src/core/lib/gprpp/env.h" #include "src/core/lib/gprpp/host_port.h" @@ -258,11 +260,17 @@ absl::optional HttpProxyMapper::MapName( MaybeAddDefaultPort(absl::StripPrefix(uri->path(), "/"))); if (user_cred.has_value()) { // Use base64 encoding for user credentials as stated in RFC 7617 - auto encoded_user_cred = UniquePtr( - grpc_base64_encode(user_cred->data(), user_cred->length(), 0, 0)); + std::string encoded_user_cred; + if (IsAbslBase64Enabled()) { + encoded_user_cred = absl::Base64Escape(*user_cred); + } else { + UniquePtr tmp( + grpc_base64_encode(user_cred->data(), user_cred->length(), 0, 0)); + encoded_user_cred = tmp.get(); + } *args = args->Set( GRPC_ARG_HTTP_CONNECT_HEADERS, - absl::StrCat("Proxy-Authorization:Basic ", encoded_user_cred.get())); + absl::StrCat("Proxy-Authorization:Basic ", encoded_user_cred)); } return name_to_resolve; } diff --git a/src/core/lib/experiments/experiments.cc b/src/core/lib/experiments/experiments.cc index f1eef9869ca..ac4dcd85889 100644 --- a/src/core/lib/experiments/experiments.cc +++ b/src/core/lib/experiments/experiments.cc @@ -24,6 +24,8 @@ #if defined(GRPC_CFSTREAM) namespace { +const char* const description_absl_base64 = "Use abseil base64 functions."; +const char* const additional_constraints_absl_base64 = "{}"; const char* const description_call_status_override_on_cancellation = "Avoid overriding call status of successfully finished calls if it races " "with cancellation."; @@ -193,6 +195,8 @@ const bool kDefaultForDebugOnly = true; namespace grpc_core { const ExperimentMetadata g_experiment_metadata[] = { + {"absl_base64", description_absl_base64, additional_constraints_absl_base64, + nullptr, 0, true, true}, {"call_status_override_on_cancellation", description_call_status_override_on_cancellation, additional_constraints_call_status_override_on_cancellation, nullptr, 0, @@ -297,6 +301,8 @@ const ExperimentMetadata g_experiment_metadata[] = { #elif defined(GPR_WINDOWS) namespace { +const char* const description_absl_base64 = "Use abseil base64 functions."; +const char* const additional_constraints_absl_base64 = "{}"; const char* const description_call_status_override_on_cancellation = "Avoid overriding call status of successfully finished calls if it races " "with cancellation."; @@ -466,6 +472,8 @@ const bool kDefaultForDebugOnly = true; namespace grpc_core { const ExperimentMetadata g_experiment_metadata[] = { + {"absl_base64", description_absl_base64, additional_constraints_absl_base64, + nullptr, 0, true, true}, {"call_status_override_on_cancellation", description_call_status_override_on_cancellation, additional_constraints_call_status_override_on_cancellation, nullptr, 0, @@ -570,6 +578,8 @@ const ExperimentMetadata g_experiment_metadata[] = { #else namespace { +const char* const description_absl_base64 = "Use abseil base64 functions."; +const char* const additional_constraints_absl_base64 = "{}"; const char* const description_call_status_override_on_cancellation = "Avoid overriding call status of successfully finished calls if it races " "with cancellation."; @@ -739,6 +749,8 @@ const bool kDefaultForDebugOnly = true; namespace grpc_core { const ExperimentMetadata g_experiment_metadata[] = { + {"absl_base64", description_absl_base64, additional_constraints_absl_base64, + nullptr, 0, true, true}, {"call_status_override_on_cancellation", description_call_status_override_on_cancellation, additional_constraints_call_status_override_on_cancellation, nullptr, 0, diff --git a/src/core/lib/experiments/experiments.h b/src/core/lib/experiments/experiments.h index 778a99f9222..af3b099429d 100644 --- a/src/core/lib/experiments/experiments.h +++ b/src/core/lib/experiments/experiments.h @@ -57,6 +57,8 @@ namespace grpc_core { #ifdef GRPC_EXPERIMENTS_ARE_FINAL #if defined(GRPC_CFSTREAM) +#define GRPC_EXPERIMENT_IS_INCLUDED_ABSL_BASE64 +inline bool IsAbslBase64Enabled() { return true; } #ifndef NDEBUG #define GRPC_EXPERIMENT_IS_INCLUDED_CALL_STATUS_OVERRIDE_ON_CANCELLATION #endif @@ -119,6 +121,8 @@ inline bool IsWriteSizeCapEnabled() { return true; } inline bool IsWrrDelegateToPickFirstEnabled() { return true; } #elif defined(GPR_WINDOWS) +#define GRPC_EXPERIMENT_IS_INCLUDED_ABSL_BASE64 +inline bool IsAbslBase64Enabled() { return true; } #ifndef NDEBUG #define GRPC_EXPERIMENT_IS_INCLUDED_CALL_STATUS_OVERRIDE_ON_CANCELLATION #endif @@ -182,6 +186,8 @@ inline bool IsWriteSizeCapEnabled() { return true; } inline bool IsWrrDelegateToPickFirstEnabled() { return true; } #else +#define GRPC_EXPERIMENT_IS_INCLUDED_ABSL_BASE64 +inline bool IsAbslBase64Enabled() { return true; } #ifndef NDEBUG #define GRPC_EXPERIMENT_IS_INCLUDED_CALL_STATUS_OVERRIDE_ON_CANCELLATION #endif @@ -247,6 +253,7 @@ inline bool IsWrrDelegateToPickFirstEnabled() { return true; } #else enum ExperimentIds { + kExperimentIdAbslBase64, kExperimentIdCallStatusOverrideOnCancellation, kExperimentIdCallV3, kExperimentIdCanaryClientPrivacy, @@ -289,6 +296,10 @@ enum ExperimentIds { kExperimentIdWrrDelegateToPickFirst, kNumExperiments }; +#define GRPC_EXPERIMENT_IS_INCLUDED_ABSL_BASE64 +inline bool IsAbslBase64Enabled() { + return IsExperimentEnabled(kExperimentIdAbslBase64); +} #define GRPC_EXPERIMENT_IS_INCLUDED_CALL_STATUS_OVERRIDE_ON_CANCELLATION inline bool IsCallStatusOverrideOnCancellationEnabled() { return IsExperimentEnabled(kExperimentIdCallStatusOverrideOnCancellation); diff --git a/src/core/lib/experiments/experiments.yaml b/src/core/lib/experiments/experiments.yaml index 283802ffc45..74b2d14eada 100644 --- a/src/core/lib/experiments/experiments.yaml +++ b/src/core/lib/experiments/experiments.yaml @@ -40,6 +40,11 @@ # This file only defines the experiments. Refer to rollouts.yaml for the rollout # state of each experiment. +- name: absl_base64 + description: Use abseil base64 functions. + expiry: 2024/06/01 + owner: roth@google.com + test_tags: ["credential_token_tests"] - name: call_status_override_on_cancellation description: Avoid overriding call status of successfully finished calls if it races with diff --git a/src/core/lib/experiments/rollouts.yaml b/src/core/lib/experiments/rollouts.yaml index 0f89e8ece39..5bea6f61e1e 100644 --- a/src/core/lib/experiments/rollouts.yaml +++ b/src/core/lib/experiments/rollouts.yaml @@ -40,6 +40,8 @@ # # Supported platforms: ios, windows, posix +- name: absl_base64 + default: true - name: call_status_override_on_cancellation default: debug - name: call_v3 diff --git a/src/core/lib/security/credentials/jwt/json_token.cc b/src/core/lib/security/credentials/jwt/json_token.cc index 47eac88aafe..dd34cbb1dcc 100644 --- a/src/core/lib/security/credentials/jwt/json_token.cc +++ b/src/core/lib/security/credentials/jwt/json_token.cc @@ -33,13 +33,16 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" +#include "absl/strings/escaping.h" #include #include #include #include +#include #include +#include "src/core/lib/experiments/experiments.h" #include "src/core/lib/iomgr/error.h" #include "src/core/lib/json/json_reader.h" #include "src/core/lib/json/json_writer.h" @@ -180,7 +183,10 @@ static char* encoded_jwt_header(const char* key_id, const char* algorithm) { {"kid", Json::FromString(key_id)}, }); std::string json_str = grpc_core::JsonDump(json); - return grpc_base64_encode(json_str.c_str(), json_str.size(), 1, 0); + if (!grpc_core::IsAbslBase64Enabled()) { + return grpc_base64_encode(json_str.c_str(), json_str.size(), 1, 0); + } + return gpr_strdup(absl::WebSafeBase64Escape(json_str).c_str()); } static char* encoded_jwt_claim(const grpc_auth_json_key* json_key, @@ -208,7 +214,10 @@ static char* encoded_jwt_claim(const grpc_auth_json_key* json_key, std::string json_str = grpc_core::JsonDump(Json::FromObject(std::move(object))); - return grpc_base64_encode(json_str.c_str(), json_str.size(), 1, 0); + if (!grpc_core::IsAbslBase64Enabled()) { + return grpc_base64_encode(json_str.c_str(), json_str.size(), 1, 0); + } + return gpr_strdup(absl::WebSafeBase64Escape(json_str).c_str()); } static char* dot_concat_and_free_strings(char* str1, char* str2) { @@ -280,7 +289,14 @@ char* compute_and_encode_signature(const grpc_auth_json_key* json_key, gpr_log(GPR_ERROR, "DigestFinal (signature compute) failed."); goto end; } - result = grpc_base64_encode(sig, sig_len, 1, 0); + if (!grpc_core::IsAbslBase64Enabled()) { + result = grpc_base64_encode(sig, sig_len, 1, 0); + } else { + result = + gpr_strdup(absl::WebSafeBase64Escape( + absl::string_view(reinterpret_cast(sig), sig_len)) + .c_str()); + } end: #if OPENSSL_VERSION_NUMBER < 0x30000000L diff --git a/src/core/lib/security/credentials/jwt/jwt_verifier.cc b/src/core/lib/security/credentials/jwt/jwt_verifier.cc index 346f2491fa8..901670701c9 100644 --- a/src/core/lib/security/credentials/jwt/jwt_verifier.cc +++ b/src/core/lib/security/credentials/jwt/jwt_verifier.cc @@ -43,6 +43,7 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" +#include "absl/strings/escaping.h" #include "absl/strings/string_view.h" #include @@ -52,6 +53,7 @@ #include #include +#include "src/core/lib/experiments/experiments.h" #include "src/core/lib/gpr/string.h" #include "src/core/lib/gprpp/manual_constructor.h" #include "src/core/lib/gprpp/memory.h" @@ -111,14 +113,22 @@ static const EVP_MD* evp_md_from_alg(const char* alg) { } static Json parse_json_part_from_jwt(const char* str, size_t len) { - grpc_slice slice = grpc_base64_decode_with_len(str, len, 1); - if (GRPC_SLICE_IS_EMPTY(slice)) { - gpr_log(GPR_ERROR, "Invalid base64."); - return Json(); // JSON null + std::string string; + if (!grpc_core::IsAbslBase64Enabled()) { + grpc_slice slice = grpc_base64_decode_with_len(str, len, 1); + if (GRPC_SLICE_IS_EMPTY(slice)) { + gpr_log(GPR_ERROR, "Invalid base64."); + return Json(); // JSON null + } + string = std::string(grpc_core::StringViewFromSlice(slice)); + grpc_core::CSliceUnref(slice); + } else { + if (!absl::WebSafeBase64Unescape(absl::string_view(str, len), &string)) { + gpr_log(GPR_ERROR, "Invalid base64."); + return Json(); // JSON null + } } - absl::string_view string = grpc_core::StringViewFromSlice(slice); auto json = grpc_core::JsonParse(string); - grpc_core::CSliceUnref(slice); if (!json.ok()) { gpr_log(GPR_ERROR, "JSON parse error: %s", json.status().ToString().c_str()); @@ -480,19 +490,26 @@ end: } static BIGNUM* bignum_from_base64(const char* b64) { - BIGNUM* result = nullptr; - grpc_slice bin; - if (b64 == nullptr) return nullptr; - bin = grpc_base64_decode(b64, 1); - if (GRPC_SLICE_IS_EMPTY(bin)) { + if (!grpc_core::IsAbslBase64Enabled()) { + grpc_slice bin = grpc_base64_decode(b64, 1); + if (GRPC_SLICE_IS_EMPTY(bin)) { + gpr_log(GPR_ERROR, "Invalid base64 for big num."); + return nullptr; + } + BIGNUM* result = + BN_bin2bn(GRPC_SLICE_START_PTR(bin), + TSI_SIZE_AS_SIZE(GRPC_SLICE_LENGTH(bin)), nullptr); + grpc_core::CSliceUnref(bin); + return result; + } + std::string string; + if (!absl::WebSafeBase64Unescape(b64, &string)) { gpr_log(GPR_ERROR, "Invalid base64 for big num."); return nullptr; } - result = BN_bin2bn(GRPC_SLICE_START_PTR(bin), - TSI_SIZE_AS_SIZE(GRPC_SLICE_LENGTH(bin)), nullptr); - grpc_core::CSliceUnref(bin); - return result; + return BN_bin2bn(reinterpret_cast(string.data()), + TSI_SIZE_AS_SIZE(string.size()), nullptr); } #if OPENSSL_VERSION_NUMBER < 0x10100000L @@ -951,8 +968,15 @@ void grpc_jwt_verifier_verify(grpc_jwt_verifier* verifier, signed_jwt_len = static_cast(dot - jwt); cur = dot + 1; - signature = grpc_base64_decode(cur, 1); - if (GRPC_SLICE_IS_EMPTY(signature)) goto error; + + if (!grpc_core::IsAbslBase64Enabled()) { + signature = grpc_base64_decode(cur, 1); + if (GRPC_SLICE_IS_EMPTY(signature)) goto error; + } else { + std::string signature_str; + if (!absl::WebSafeBase64Unescape(cur, &signature_str)) goto error; + signature = grpc_slice_from_cpp_string(std::move(signature_str)); + } retrieve_key_and_verify( verifier_cb_ctx_create(verifier, pollset, header, claims, audience, signature, jwt, signed_jwt_len, user_data, cb)); diff --git a/test/core/end2end/fixtures/http_proxy_fixture.cc b/test/core/end2end/fixtures/http_proxy_fixture.cc index d2336fa5747..cc628b26d18 100644 --- a/test/core/end2end/fixtures/http_proxy_fixture.cc +++ b/test/core/end2end/fixtures/http_proxy_fixture.cc @@ -28,7 +28,9 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" +#include "absl/strings/escaping.h" #include "absl/strings/str_cat.h" +#include "absl/strings/strip.h" #include #include @@ -62,7 +64,6 @@ #include "src/core/lib/iomgr/sockaddr.h" #include "src/core/lib/iomgr/tcp_client.h" #include "src/core/lib/iomgr/tcp_server.h" -#include "src/core/lib/slice/b64.h" #include "test/core/util/port.h" struct grpc_end2end_http_proxy { @@ -490,19 +491,14 @@ static void on_server_connect_done(void* arg, grpc_error_handle error) { /// Basic /// Returns true if it matches, false otherwise /// -static bool proxy_auth_header_matches(char* proxy_auth_header_val, - char* expected_cred) { - GPR_ASSERT(proxy_auth_header_val != nullptr); - GPR_ASSERT(expected_cred != nullptr); - if (strncmp(proxy_auth_header_val, "Basic ", 6) != 0) { +static bool proxy_auth_header_matches(absl::string_view proxy_auth_header_val, + absl::string_view expected_cred) { + if (!absl::ConsumePrefix(&proxy_auth_header_val, "Basic ")) { return false; } - proxy_auth_header_val += 6; - grpc_slice decoded_slice = grpc_base64_decode(proxy_auth_header_val, 0); - const bool header_matches = - grpc_slice_str_cmp(decoded_slice, expected_cred) == 0; - grpc_slice_unref(decoded_slice); - return header_matches; + std::string decoded; + absl::Base64Unescape(proxy_auth_header_val, &decoded); + return expected_cred == decoded; } // Callback to read the HTTP CONNECT request. diff --git a/test/core/security/BUILD b/test/core/security/BUILD index 528b9841952..6b7f7bf4899 100644 --- a/test/core/security/BUILD +++ b/test/core/security/BUILD @@ -138,6 +138,7 @@ grpc_cc_test( srcs = ["json_token_test.cc"], external_deps = ["gtest"], language = "C++", + tags = ["credential_token_tests"], uses_event_engine = False, uses_polling = False, deps = [ @@ -154,6 +155,7 @@ grpc_cc_test( srcs = ["jwt_verifier_test.cc"], external_deps = ["gtest"], language = "C++", + tags = ["credential_token_tests"], uses_event_engine = False, uses_polling = False, deps = [ diff --git a/test/core/security/json_token_test.cc b/test/core/security/json_token_test.cc index 72b91d45dda..5e89cf292aa 100644 --- a/test/core/security/json_token_test.cc +++ b/test/core/security/json_token_test.cc @@ -23,6 +23,8 @@ #include #include +#include "absl/strings/escaping.h" + #include #include #include @@ -32,7 +34,6 @@ #include "src/core/lib/json/json.h" #include "src/core/lib/json/json_reader.h" #include "src/core/lib/security/credentials/oauth2/oauth2_credentials.h" -#include "src/core/lib/slice/b64.h" #include "src/core/lib/slice/slice_internal.h" #include "test/core/util/test_config.h" @@ -213,15 +214,10 @@ TEST(JsonTokenTest, ParseJsonKeyFailureNoPrivateKey) { static Json parse_json_part_from_jwt(const char* str, size_t len) { grpc_core::ExecCtx exec_ctx; - char* b64 = static_cast(gpr_malloc(len + 1)); - strncpy(b64, str, len); - b64[len] = '\0'; - grpc_slice slice = grpc_base64_decode(b64, 1); - gpr_free(b64); - EXPECT_FALSE(GRPC_SLICE_IS_EMPTY(slice)); - absl::string_view string = grpc_core::StringViewFromSlice(slice); - auto json = grpc_core::JsonParse(string); - grpc_slice_unref(slice); + std::string decoded; + absl::WebSafeBase64Unescape(absl::string_view(str, len), &decoded); + EXPECT_FALSE(decoded.empty()); + auto json = grpc_core::JsonParse(decoded); if (!json.ok()) { gpr_log(GPR_ERROR, "JSON parse error: %s", json.status().ToString().c_str()); @@ -293,9 +289,9 @@ static void check_jwt_signature(const char* b64_signature, RSA* rsa_key, EVP_MD_CTX* md_ctx = EVP_MD_CTX_create(); EVP_PKEY* key = EVP_PKEY_new(); - grpc_slice sig = grpc_base64_decode(b64_signature, 1); - ASSERT_FALSE(GRPC_SLICE_IS_EMPTY(sig)); - ASSERT_EQ(GRPC_SLICE_LENGTH(sig), 128); + std::string decoded; + absl::WebSafeBase64Unescape(b64_signature, &decoded); + ASSERT_EQ(decoded.size(), 128); ASSERT_NE(md_ctx, nullptr); ASSERT_NE(key, nullptr); @@ -304,11 +300,11 @@ static void check_jwt_signature(const char* b64_signature, RSA* rsa_key, ASSERT_EQ(EVP_DigestVerifyInit(md_ctx, nullptr, EVP_sha256(), nullptr, key), 1); ASSERT_EQ(EVP_DigestVerifyUpdate(md_ctx, signed_data, signed_data_size), 1); - ASSERT_EQ(EVP_DigestVerifyFinal(md_ctx, GRPC_SLICE_START_PTR(sig), - GRPC_SLICE_LENGTH(sig)), + ASSERT_EQ(EVP_DigestVerifyFinal( + md_ctx, reinterpret_cast(decoded.data()), + decoded.size()), 1); - grpc_slice_unref(sig); if (key != nullptr) EVP_PKEY_free(key); if (md_ctx != nullptr) EVP_MD_CTX_destroy(md_ctx); } @@ -319,18 +315,18 @@ static void check_jwt_signature(const char* b64_signature, EVP_PKEY* key, grpc_core::ExecCtx exec_ctx; EVP_MD_CTX* md_ctx = EVP_MD_CTX_create(); - grpc_slice sig = grpc_base64_decode(b64_signature, 1); - ASSERT_FALSE(GRPC_SLICE_IS_EMPTY(sig)); - ASSERT_EQ(GRPC_SLICE_LENGTH(sig), 128); + std::string decoded; + absl::WebSafeBase64Unescape(b64_signature, &decoded); + ASSERT_EQ(decoded.size(), 128); ASSERT_EQ(EVP_DigestVerifyInit(md_ctx, nullptr, EVP_sha256(), nullptr, key), 1); ASSERT_EQ(EVP_DigestVerifyUpdate(md_ctx, signed_data, signed_data_size), 1); - ASSERT_EQ(EVP_DigestVerifyFinal(md_ctx, GRPC_SLICE_START_PTR(sig), - GRPC_SLICE_LENGTH(sig)), + ASSERT_EQ(EVP_DigestVerifyFinal( + md_ctx, reinterpret_cast(decoded.data()), + decoded.size()), 1); - grpc_slice_unref(sig); if (md_ctx != nullptr) EVP_MD_CTX_destroy(md_ctx); } #endif diff --git a/test/core/security/jwt_verifier_test.cc b/test/core/security/jwt_verifier_test.cc index 3aa42efd52a..6065e7aef82 100644 --- a/test/core/security/jwt_verifier_test.cc +++ b/test/core/security/jwt_verifier_test.cc @@ -22,6 +22,8 @@ #include +#include "absl/strings/escaping.h" + #include #include #include @@ -32,7 +34,6 @@ #include "src/core/lib/http/httpcli.h" #include "src/core/lib/json/json_reader.h" #include "src/core/lib/security/credentials/jwt/json_token.h" -#include "src/core/lib/slice/b64.h" #include "test/core/util/test_config.h" using grpc_core::Json; @@ -536,23 +537,14 @@ TEST(JwtVerifierTest, JwtVerifierBadJsonKey) { } static void corrupt_jwt_sig(char* jwt) { - grpc_slice sig; - char* bad_b64_sig; - uint8_t* sig_bytes; char* last_dot = strrchr(jwt, '.'); ASSERT_NE(last_dot, nullptr); - { - grpc_core::ExecCtx exec_ctx; - sig = grpc_base64_decode(last_dot + 1, 1); - } - ASSERT_FALSE(GRPC_SLICE_IS_EMPTY(sig)); - sig_bytes = GRPC_SLICE_START_PTR(sig); - (*sig_bytes)++; // Corrupt first byte. - bad_b64_sig = grpc_base64_encode(GRPC_SLICE_START_PTR(sig), - GRPC_SLICE_LENGTH(sig), 1, 0); - memcpy(last_dot + 1, bad_b64_sig, strlen(bad_b64_sig)); - gpr_free(bad_b64_sig); - grpc_slice_unref(sig); + std::string decoded; + absl::WebSafeBase64Unescape(last_dot + 1, &decoded); + ASSERT_FALSE(decoded.empty()); + ++decoded[0]; // Corrupt first byte. + std::string bad_encoding = absl::WebSafeBase64Escape(decoded); + memcpy(last_dot + 1, bad_encoding.data(), bad_encoding.size()); } static void on_verification_bad_signature(void* user_data, diff --git a/test/cpp/interop/client_helper.cc b/test/cpp/interop/client_helper.cc index 225d598caae..374eb4085d3 100644 --- a/test/cpp/interop/client_helper.cc +++ b/test/cpp/interop/client_helper.cc @@ -24,6 +24,7 @@ #include "absl/flags/declare.h" #include "absl/flags/flag.h" +#include "absl/strings/escaping.h" #include "absl/strings/match.h" #include @@ -34,7 +35,6 @@ #include #include "src/core/lib/gprpp/crash.h" -#include "src/core/lib/slice/b64.h" #include "src/cpp/client/secure_credentials.h" #include "test/core/security/oauth2_utils.h" #include "test/cpp/util/create_test_channel.h" @@ -146,13 +146,10 @@ std::shared_ptr CreateChannelForTestCase( static void log_metadata_entry(const std::string& prefix, const grpc::string_ref& key, const grpc::string_ref& value) { - auto key_str = std::string(key.begin(), key.end()); - auto value_str = std::string(value.begin(), value.end()); + std::string key_str(key.begin(), key.end()); + std::string value_str(value.begin(), value.end()); if (absl::EndsWith(key_str, "-bin")) { - auto converted = - grpc_base64_encode(value_str.c_str(), value_str.length(), 0, 0); - value_str = std::string(converted); - gpr_free(converted); + value_str = absl::Base64Escape(value_str); } gpr_log(GPR_ERROR, "%s %s: %s", prefix.c_str(), key_str.c_str(), value_str.c_str()); From 574b0572f1dbd60d687bef3354dc1141f067b374 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Tue, 13 Feb 2024 15:28:34 -0800 Subject: [PATCH 13/37] [client channel] remove grpc_channel_num_external_connectivity_watchers() (#35840) Implements gRFC L113 (https://github.com/grpc/proposal/pull/417). Closes #35840 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/35840 from markdroth:client_channel_remove_num_external_watchers 334670c13c29b754a2483f25678ade2ce6bf1bfb PiperOrigin-RevId: 606766495 --- grpc.def | 1 - include/grpc/grpc.h | 6 ------ src/core/client_channel/channel_connectivity.cc | 15 --------------- src/core/client_channel/client_channel_filter.h | 5 ----- src/ruby/ext/grpc/rb_grpc_imports.generated.c | 2 -- src/ruby/ext/grpc/rb_grpc_imports.generated.h | 3 --- test/core/surface/concurrent_connectivity_test.cc | 4 ---- .../num_external_connectivity_watchers_test.cc | 14 +++----------- test/core/surface/sequential_connectivity_test.cc | 3 --- 9 files changed, 3 insertions(+), 50 deletions(-) diff --git a/grpc.def b/grpc.def index 0cc981d0fd5..41ad2dd61df 100644 --- a/grpc.def +++ b/grpc.def @@ -41,7 +41,6 @@ EXPORTS grpc_completion_queue_thread_local_cache_init grpc_completion_queue_thread_local_cache_flush grpc_channel_check_connectivity_state - grpc_channel_num_external_connectivity_watchers grpc_channel_watch_connectivity_state grpc_channel_support_connectivity_watcher grpc_channel_create_call diff --git a/include/grpc/grpc.h b/include/grpc/grpc.h index d03f8a7df0c..ad475897b17 100644 --- a/include/grpc/grpc.h +++ b/include/grpc/grpc.h @@ -177,12 +177,6 @@ GRPCAPI int grpc_completion_queue_thread_local_cache_flush( GRPCAPI grpc_connectivity_state grpc_channel_check_connectivity_state( grpc_channel* channel, int try_to_connect); -/** Number of active "external connectivity state watchers" attached to a - * channel. - * Useful for testing. **/ -GRPCAPI int grpc_channel_num_external_connectivity_watchers( - grpc_channel* channel); - /** Watch for a change in connectivity state. Once the channel connectivity state is different from last_observed_state, tag will be enqueued on cq with success=1. diff --git a/src/core/client_channel/channel_connectivity.cc b/src/core/client_channel/channel_connectivity.cc index 52fc35a9ba0..5ae94d612a5 100644 --- a/src/core/client_channel/channel_connectivity.cc +++ b/src/core/client_channel/channel_connectivity.cc @@ -81,21 +81,6 @@ grpc_connectivity_state grpc_channel_check_connectivity_state( return client_channel->CheckConnectivityState(try_to_connect); } -int grpc_channel_num_external_connectivity_watchers(grpc_channel* c_channel) { - grpc_core::Channel* channel = grpc_core::Channel::FromC(c_channel); - grpc_core::ClientChannelFilter* client_channel = - grpc_core::ClientChannelFilter::GetFromChannel(channel); - if (client_channel == nullptr) { - if (!grpc_core::IsLameChannel(channel)) { - gpr_log(GPR_ERROR, - "grpc_channel_num_external_connectivity_watchers called on " - "something that is not a client channel"); - } - return 0; - } - return client_channel->NumExternalConnectivityWatchers(); -} - int grpc_channel_support_connectivity_watcher(grpc_channel* channel) { return grpc_core::ClientChannelFilter::GetFromChannel( grpc_core::Channel::FromC(channel)) != nullptr; diff --git a/src/core/client_channel/client_channel_filter.h b/src/core/client_channel/client_channel_filter.h index e27178b4a67..c904c8f23e2 100644 --- a/src/core/client_channel/client_channel_filter.h +++ b/src/core/client_channel/client_channel_filter.h @@ -149,11 +149,6 @@ class ClientChannelFilter { this, on_complete, /*cancel=*/true); } - int NumExternalConnectivityWatchers() const { - MutexLock lock(&external_watchers_mu_); - return static_cast(external_watchers_.size()); - } - // Starts and stops a connectivity watch. The watcher will be initially // notified as soon as the state changes from initial_state and then on // every subsequent state change until either the watch is stopped or diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.c b/src/ruby/ext/grpc/rb_grpc_imports.generated.c index b28175e0165..302dbf49f44 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.c +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.c @@ -64,7 +64,6 @@ grpc_completion_queue_destroy_type grpc_completion_queue_destroy_import; grpc_completion_queue_thread_local_cache_init_type grpc_completion_queue_thread_local_cache_init_import; grpc_completion_queue_thread_local_cache_flush_type grpc_completion_queue_thread_local_cache_flush_import; grpc_channel_check_connectivity_state_type grpc_channel_check_connectivity_state_import; -grpc_channel_num_external_connectivity_watchers_type grpc_channel_num_external_connectivity_watchers_import; grpc_channel_watch_connectivity_state_type grpc_channel_watch_connectivity_state_import; grpc_channel_support_connectivity_watcher_type grpc_channel_support_connectivity_watcher_import; grpc_channel_create_call_type grpc_channel_create_call_import; @@ -355,7 +354,6 @@ void grpc_rb_load_imports(HMODULE library) { grpc_completion_queue_thread_local_cache_init_import = (grpc_completion_queue_thread_local_cache_init_type) GetProcAddress(library, "grpc_completion_queue_thread_local_cache_init"); grpc_completion_queue_thread_local_cache_flush_import = (grpc_completion_queue_thread_local_cache_flush_type) GetProcAddress(library, "grpc_completion_queue_thread_local_cache_flush"); grpc_channel_check_connectivity_state_import = (grpc_channel_check_connectivity_state_type) GetProcAddress(library, "grpc_channel_check_connectivity_state"); - grpc_channel_num_external_connectivity_watchers_import = (grpc_channel_num_external_connectivity_watchers_type) GetProcAddress(library, "grpc_channel_num_external_connectivity_watchers"); grpc_channel_watch_connectivity_state_import = (grpc_channel_watch_connectivity_state_type) GetProcAddress(library, "grpc_channel_watch_connectivity_state"); grpc_channel_support_connectivity_watcher_import = (grpc_channel_support_connectivity_watcher_type) GetProcAddress(library, "grpc_channel_support_connectivity_watcher"); grpc_channel_create_call_import = (grpc_channel_create_call_type) GetProcAddress(library, "grpc_channel_create_call"); diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h index 65c095a96e4..d4e2099eda4 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h @@ -167,9 +167,6 @@ extern grpc_completion_queue_thread_local_cache_flush_type grpc_completion_queue typedef grpc_connectivity_state(*grpc_channel_check_connectivity_state_type)(grpc_channel* channel, int try_to_connect); extern grpc_channel_check_connectivity_state_type grpc_channel_check_connectivity_state_import; #define grpc_channel_check_connectivity_state grpc_channel_check_connectivity_state_import -typedef int(*grpc_channel_num_external_connectivity_watchers_type)(grpc_channel* channel); -extern grpc_channel_num_external_connectivity_watchers_type grpc_channel_num_external_connectivity_watchers_import; -#define grpc_channel_num_external_connectivity_watchers grpc_channel_num_external_connectivity_watchers_import typedef void(*grpc_channel_watch_connectivity_state_type)(grpc_channel* channel, grpc_connectivity_state last_observed_state, gpr_timespec deadline, grpc_completion_queue* cq, void* tag); extern grpc_channel_watch_connectivity_state_type grpc_channel_watch_connectivity_state_import; #define grpc_channel_watch_connectivity_state grpc_channel_watch_connectivity_state_import diff --git a/test/core/surface/concurrent_connectivity_test.cc b/test/core/surface/concurrent_connectivity_test.cc index abf0ac7cd22..2b1ee72dc94 100644 --- a/test/core/surface/concurrent_connectivity_test.cc +++ b/test/core/surface/concurrent_connectivity_test.cc @@ -93,8 +93,6 @@ void create_loop_destroy(void* addr) { grpc_timeout_milliseconds_to_deadline(POLL_MILLIS); ASSERT_EQ(grpc_completion_queue_next(cq, poll_time, nullptr).type, GRPC_OP_COMPLETE); - // check that the watcher from "watch state" was free'd - ASSERT_EQ(grpc_channel_num_external_connectivity_watchers(chan), 0); } grpc_channel_destroy(chan); grpc_completion_queue_destroy(cq); @@ -292,8 +290,6 @@ void watches_with_short_timeouts(void* addr) { grpc_event ev = grpc_completion_queue_next(cq, poll_time, nullptr); ASSERT_EQ(ev.type, GRPC_OP_COMPLETE); ASSERT_EQ(ev.success, false); - // check that the watcher from "watch state" was free'd - ASSERT_EQ(grpc_channel_num_external_connectivity_watchers(chan), 0); } grpc_channel_destroy(chan); grpc_completion_queue_destroy(cq); diff --git a/test/core/surface/num_external_connectivity_watchers_test.cc b/test/core/surface/num_external_connectivity_watchers_test.cc index bf0178b6ffc..40ca9b67385 100644 --- a/test/core/surface/num_external_connectivity_watchers_test.cc +++ b/test/core/surface/num_external_connectivity_watchers_test.cc @@ -55,8 +55,6 @@ static void channel_idle_start_watch(grpc_channel* channel, grpc_channel_watch_connectivity_state(channel, GRPC_CHANNEL_IDLE, connect_deadline, cq, reinterpret_cast(next_tag++)); - gpr_log(GPR_DEBUG, "number of active connect watchers: %d", - grpc_channel_num_external_connectivity_watchers(channel)); } static void channel_idle_poll_for_timeout(grpc_channel* channel, @@ -71,9 +69,8 @@ static void channel_idle_poll_for_timeout(grpc_channel* channel, GRPC_CHANNEL_IDLE); } -// Test and use the "num_external_watchers" call to make sure -// that "connectivity watcher" structs are free'd just after, if -// their corresponding timeouts occur. +// Test to make sure that "connectivity watcher" structs are free'd just +// after, if their corresponding timeouts occur. static void run_timeouts_test(const test_fixture* fixture) { gpr_log(GPR_INFO, "TEST: %s", fixture->name); @@ -87,7 +84,6 @@ static void run_timeouts_test(const test_fixture* fixture) { // start 1 watcher and then let it time out channel_idle_start_watch(channel, cq); channel_idle_poll_for_timeout(channel, cq); - ASSERT_EQ(grpc_channel_num_external_connectivity_watchers(channel), 0); // start 3 watchers and then let them all time out for (size_t i = 1; i <= 3; i++) { @@ -96,7 +92,6 @@ static void run_timeouts_test(const test_fixture* fixture) { for (size_t i = 1; i <= 3; i++) { channel_idle_poll_for_timeout(channel, cq); } - ASSERT_EQ(grpc_channel_num_external_connectivity_watchers(channel), 0); // start 3 watchers, see one time out, start another 3, and then see them all // time out @@ -110,7 +105,6 @@ static void run_timeouts_test(const test_fixture* fixture) { for (size_t i = 1; i <= 5; i++) { channel_idle_poll_for_timeout(channel, cq); } - ASSERT_EQ(grpc_channel_num_external_connectivity_watchers(channel), 0); grpc_channel_destroy(channel); grpc_completion_queue_shutdown(cq); @@ -136,9 +130,7 @@ static void run_channel_shutdown_before_timeout_test( grpc_channel* channel = fixture->create_channel(addr.c_str()); grpc_completion_queue* cq = grpc_completion_queue_create_for_next(nullptr); - // start 1 watcher and then shut down the channel before the timer goes off - ASSERT_EQ(grpc_channel_num_external_connectivity_watchers(channel), 0); - + // start 1 watcher and then shut down the channel before the timer goes off. // expecting a 30 second timeout to go off much later than the shutdown. gpr_timespec connect_deadline = grpc_timeout_seconds_to_deadline(30); ASSERT_EQ(grpc_channel_check_connectivity_state(channel, 0), diff --git a/test/core/surface/sequential_connectivity_test.cc b/test/core/surface/sequential_connectivity_test.cc index 6efaa787e8d..73a053e1196 100644 --- a/test/core/surface/sequential_connectivity_test.cc +++ b/test/core/surface/sequential_connectivity_test.cc @@ -124,9 +124,6 @@ static void run_test(const test_fixture* fixture, bool share_subchannel) { connect_deadline, cq, nullptr); grpc_event ev = grpc_completion_queue_next( cq, gpr_inf_future(GPR_CLOCK_REALTIME), nullptr); - // check that the watcher from "watch state" was free'd - ASSERT_EQ(grpc_channel_num_external_connectivity_watchers(channels[i]), - 0); ASSERT_EQ(ev.type, GRPC_OP_COMPLETE); ASSERT_EQ(ev.tag, nullptr); ASSERT_EQ(ev.success, true); From 6c157f9128a17f5fb9b065750969506bb4814b9f Mon Sep 17 00:00:00 2001 From: Eugene Ostroukhov Date: Tue, 13 Feb 2024 16:05:54 -0800 Subject: [PATCH 14/37] [Release] Bump version to 1.63.0-dev (on master branch) (#35899) Change was created by the release automation script. See go/grpc-release. Closes #35899 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/35899 from eugeneo:bump_dev_version_202402131133 73950425c9f00d56ce8d6b9f2f36f69b6dce4a40 PiperOrigin-RevId: 606777850 --- BUILD | 4 ++-- CMakeLists.txt | 6 +++--- Makefile | 2 +- _metadata.py | 2 +- build_handwritten.yaml | 4 ++-- config.m4 | 2 +- doc/g_stands_for.md | 3 ++- gRPC-C++.podspec | 2 +- gRPC-Core.podspec | 2 +- gRPC-ProtoRPC.podspec | 2 +- gRPC-RxLibrary.podspec | 2 +- gRPC.podspec | 2 +- include/grpcpp/version_info.h | 4 ++-- package.xml | 6 +++--- src/core/lib/surface/version.cc | 2 +- src/csharp/build/dependencies.props | 2 +- src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec | 2 +- src/objective-c/!ProtoCompiler-gRPCPlugin.podspec | 2 +- src/objective-c/GRPCClient/version.h | 2 +- src/objective-c/tests/version.h | 2 +- src/php/composer.json | 2 +- src/php/ext/grpc/version.h | 2 +- src/python/grpcio/grpc/_grpcio_metadata.py | 2 +- src/python/grpcio/grpc_version.py | 2 +- src/python/grpcio_admin/grpc_version.py | 2 +- src/python/grpcio_channelz/grpc_version.py | 2 +- src/python/grpcio_csds/grpc_version.py | 2 +- src/python/grpcio_health_checking/grpc_version.py | 2 +- src/python/grpcio_observability/grpc_version.py | 2 +- src/python/grpcio_reflection/grpc_version.py | 2 +- src/python/grpcio_status/grpc_version.py | 2 +- src/python/grpcio_testing/grpc_version.py | 2 +- src/python/grpcio_tests/grpc_version.py | 2 +- src/ruby/lib/grpc/version.rb | 2 +- src/ruby/nativedebug/version.rb | 2 +- src/ruby/tools/version.rb | 2 +- tools/distrib/python/grpc_version.py | 2 +- tools/distrib/python/grpcio_tools/grpc_version.py | 2 +- tools/distrib/python/xds_protos/grpc_version.py | 2 +- tools/doxygen/Doxyfile.c++ | 2 +- tools/doxygen/Doxyfile.c++.internal | 2 +- tools/doxygen/Doxyfile.objc | 2 +- tools/doxygen/Doxyfile.objc.internal | 2 +- tools/doxygen/Doxyfile.php | 2 +- 44 files changed, 52 insertions(+), 51 deletions(-) diff --git a/BUILD b/BUILD index 747cf55f4c1..236a89ec982 100644 --- a/BUILD +++ b/BUILD @@ -211,11 +211,11 @@ config_setting( python_config_settings() # This should be updated along with build_handwritten.yaml -g_stands_for = "guardian" # @unused +g_stands_for = "giggle" # @unused core_version = "39.0.0" # @unused -version = "1.62.0-dev" # @unused +version = "1.63.0-dev" # @unused GPR_PUBLIC_HDRS = [ "include/grpc/support/alloc.h", diff --git a/CMakeLists.txt b/CMakeLists.txt index fe9b7314195..e1cdd443aff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,11 +25,11 @@ cmake_minimum_required(VERSION 3.13) set(PACKAGE_NAME "grpc") -set(PACKAGE_VERSION "1.62.0-dev") +set(PACKAGE_VERSION "1.63.0-dev") set(gRPC_CORE_VERSION "39.0.0") set(gRPC_CORE_SOVERSION "39") -set(gRPC_CPP_VERSION "1.62.0-dev") -set(gRPC_CPP_SOVERSION "1.62") +set(gRPC_CPP_VERSION "1.63.0-dev") +set(gRPC_CPP_SOVERSION "1.63") set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}") set(PACKAGE_TARNAME "${PACKAGE_NAME}-${PACKAGE_VERSION}") set(PACKAGE_BUGREPORT "https://github.com/grpc/grpc/issues/") diff --git a/Makefile b/Makefile index e268990e44b..9fe4e2b80b1 100644 --- a/Makefile +++ b/Makefile @@ -411,7 +411,7 @@ Q = @ endif CORE_VERSION = 39.0.0 -CPP_VERSION = 1.62.0-dev +CPP_VERSION = 1.63.0-dev CPPFLAGS_NO_ARCH += $(addprefix -I, $(INCLUDES)) $(addprefix -D, $(DEFINES)) CPPFLAGS += $(CPPFLAGS_NO_ARCH) $(ARCH_FLAGS) diff --git a/_metadata.py b/_metadata.py index da74eea2ed3..d03cbf8858a 100644 --- a/_metadata.py +++ b/_metadata.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/_metadata.py.template`!!! -__version__ = """1.62.0.dev0""" +__version__ = """1.63.0.dev0""" diff --git a/build_handwritten.yaml b/build_handwritten.yaml index ced47463e95..6282f4993dc 100644 --- a/build_handwritten.yaml +++ b/build_handwritten.yaml @@ -14,9 +14,9 @@ settings: '#10': See the expand_version.py for all the quirks here core_version: 39.0.0 csharp_major_version: 2 - g_stands_for: guardian + g_stands_for: giggle protobuf_version: 3.25.1 - version: 1.62.0-dev + version: 1.63.0-dev configs: asan: CC: clang diff --git a/config.m4 b/config.m4 index dcb1c73b8ef..97c6f517823 100644 --- a/config.m4 +++ b/config.m4 @@ -1368,7 +1368,7 @@ if test "$PHP_GRPC" != "no"; then -D_HAS_EXCEPTIONS=0 -DNOMINMAX -DGRPC_ARES=0 \ -DGRPC_POSIX_FORK_ALLOW_PTHREAD_ATFORK=1 \ -DGRPC_XDS_USER_AGENT_NAME_SUFFIX='"\"PHP\""' \ - -DGRPC_XDS_USER_AGENT_VERSION_SUFFIX='"\"1.62.0dev\""') + -DGRPC_XDS_USER_AGENT_VERSION_SUFFIX='"\"1.63.0dev\""') PHP_ADD_BUILD_DIR($ext_builddir/src/core/client_channel) PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/backend_metrics) diff --git a/doc/g_stands_for.md b/doc/g_stands_for.md index 37119622645..a10dc1c0c95 100644 --- a/doc/g_stands_for.md +++ b/doc/g_stands_for.md @@ -61,4 +61,5 @@ - 1.59 'g' stands for ['generative'](https://github.com/grpc/grpc/tree/v1.59.x) - 1.60 'g' stands for ['gjallarhorn'](https://github.com/grpc/grpc/tree/v1.60.x) - 1.61 'g' stands for ['grand'](https://github.com/grpc/grpc/tree/v1.61.x) -- 1.62 'g' stands for ['guardian'](https://github.com/grpc/grpc/tree/master) +- 1.62 'g' stands for ['guardian'](https://github.com/grpc/grpc/tree/v1.62.x) +- 1.63 'g' stands for ['giggle'](https://github.com/grpc/grpc/tree/master) diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index 2429d8100ac..bdfe95324ae 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -22,7 +22,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-C++' # TODO (mxyan): use version that match gRPC version when pod is stabilized - version = '1.62.0-dev' + version = '1.63.0-dev' s.version = version s.summary = 'gRPC C++ library' s.homepage = 'https://grpc.io' diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index df11c7eb828..e7989599bb6 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-Core' - version = '1.62.0-dev' + version = '1.63.0-dev' s.version = version s.summary = 'Core cross-platform gRPC library, written in C' s.homepage = 'https://grpc.io' diff --git a/gRPC-ProtoRPC.podspec b/gRPC-ProtoRPC.podspec index fe317171812..c5dba8b3efd 100644 --- a/gRPC-ProtoRPC.podspec +++ b/gRPC-ProtoRPC.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-ProtoRPC' - version = '1.62.0-dev' + version = '1.63.0-dev' s.version = version s.summary = 'RPC library for Protocol Buffers, based on gRPC' s.homepage = 'https://grpc.io' diff --git a/gRPC-RxLibrary.podspec b/gRPC-RxLibrary.podspec index a6895abe0e1..558dcda0278 100644 --- a/gRPC-RxLibrary.podspec +++ b/gRPC-RxLibrary.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-RxLibrary' - version = '1.62.0-dev' + version = '1.63.0-dev' s.version = version s.summary = 'Reactive Extensions library for iOS/OSX.' s.homepage = 'https://grpc.io' diff --git a/gRPC.podspec b/gRPC.podspec index 896e67787d1..f60642522d2 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -20,7 +20,7 @@ Pod::Spec.new do |s| s.name = 'gRPC' - version = '1.62.0-dev' + version = '1.63.0-dev' s.version = version s.summary = 'gRPC client library for iOS/OSX' s.homepage = 'https://grpc.io' diff --git a/include/grpcpp/version_info.h b/include/grpcpp/version_info.h index 4755791d373..d6dadc83b29 100644 --- a/include/grpcpp/version_info.h +++ b/include/grpcpp/version_info.h @@ -19,9 +19,9 @@ #define GRPCPP_VERSION_INFO_H #define GRPC_CPP_VERSION_MAJOR 1 -#define GRPC_CPP_VERSION_MINOR 62 +#define GRPC_CPP_VERSION_MINOR 63 #define GRPC_CPP_VERSION_PATCH 0 #define GRPC_CPP_VERSION_TAG "dev" -#define GRPC_CPP_VERSION_STRING "1.62.0-dev" +#define GRPC_CPP_VERSION_STRING "1.63.0-dev" #endif // GRPCPP_VERSION_INFO_H diff --git a/package.xml b/package.xml index 070e57ecb2b..a0dde5572de 100644 --- a/package.xml +++ b/package.xml @@ -13,8 +13,8 @@ 2019-09-24 - 1.62.0dev - 1.62.0dev + 1.63.0dev + 1.63.0dev beta @@ -22,7 +22,7 @@ Apache 2.0 -- gRPC Core 1.62.0 update +- gRPC Core 1.63.0 update diff --git a/src/core/lib/surface/version.cc b/src/core/lib/surface/version.cc index f27659c6e57..464d251db93 100644 --- a/src/core/lib/surface/version.cc +++ b/src/core/lib/surface/version.cc @@ -25,4 +25,4 @@ const char* grpc_version_string(void) { return "39.0.0"; } -const char* grpc_g_stands_for(void) { return "guardian"; } +const char* grpc_g_stands_for(void) { return "giggle"; } diff --git a/src/csharp/build/dependencies.props b/src/csharp/build/dependencies.props index 86810eb3239..fe342fc2a3d 100644 --- a/src/csharp/build/dependencies.props +++ b/src/csharp/build/dependencies.props @@ -1,7 +1,7 @@ - 2.62.0-dev + 2.63.0-dev 3.25.1 diff --git a/src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec index c254d285738..73546feac3b 100644 --- a/src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec @@ -42,7 +42,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler-gRPCCppPlugin' - v = '1.62.0-dev' + v = '1.63.0-dev' s.version = v s.summary = 'The gRPC ProtoC plugin generates C++ files from .proto services.' s.description = <<-DESC diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index 7352770f869..f282fc9fffb 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -42,7 +42,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler-gRPCPlugin' - v = '1.62.0-dev' + v = '1.63.0-dev' s.version = v s.summary = 'The gRPC ProtoC plugin generates Objective-C files from .proto services.' s.description = <<-DESC diff --git a/src/objective-c/GRPCClient/version.h b/src/objective-c/GRPCClient/version.h index 45788b886e9..4afa8ec15c0 100644 --- a/src/objective-c/GRPCClient/version.h +++ b/src/objective-c/GRPCClient/version.h @@ -22,4 +22,4 @@ // instead. This file can be regenerated from the template by running // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.62.0-dev" +#define GRPC_OBJC_VERSION_STRING @"1.63.0-dev" diff --git a/src/objective-c/tests/version.h b/src/objective-c/tests/version.h index 135cb772c77..05d1d5d45e2 100644 --- a/src/objective-c/tests/version.h +++ b/src/objective-c/tests/version.h @@ -22,5 +22,5 @@ // instead. This file can be regenerated from the template by running // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.62.0-dev" +#define GRPC_OBJC_VERSION_STRING @"1.63.0-dev" #define GRPC_C_VERSION_STRING @"39.0.0" diff --git a/src/php/composer.json b/src/php/composer.json index c61b94808cf..b8b846d2b45 100644 --- a/src/php/composer.json +++ b/src/php/composer.json @@ -2,7 +2,7 @@ "name": "grpc/grpc-dev", "description": "gRPC library for PHP - for Development use only", "license": "Apache-2.0", - "version": "1.62.0", + "version": "1.63.0", "require": { "php": ">=7.0.0", "google/protobuf": "^v3.3.0" diff --git a/src/php/ext/grpc/version.h b/src/php/ext/grpc/version.h index ad89b5f520f..45765323e22 100644 --- a/src/php/ext/grpc/version.h +++ b/src/php/ext/grpc/version.h @@ -20,6 +20,6 @@ #ifndef VERSION_H #define VERSION_H -#define PHP_GRPC_VERSION "1.62.0dev" +#define PHP_GRPC_VERSION "1.63.0dev" #endif /* VERSION_H */ diff --git a/src/python/grpcio/grpc/_grpcio_metadata.py b/src/python/grpcio/grpc/_grpcio_metadata.py index 7faef76a024..72c9c3deba4 100644 --- a/src/python/grpcio/grpc/_grpcio_metadata.py +++ b/src/python/grpcio/grpc/_grpcio_metadata.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc/_grpcio_metadata.py.template`!!! -__version__ = """1.62.0.dev0""" +__version__ = """1.63.0.dev0""" diff --git a/src/python/grpcio/grpc_version.py b/src/python/grpcio/grpc_version.py index 468946756c1..14288f6944b 100644 --- a/src/python/grpcio/grpc_version.py +++ b/src/python/grpcio/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_version.py.template`!!! -VERSION = '1.62.0.dev0' +VERSION = '1.63.0.dev0' diff --git a/src/python/grpcio_admin/grpc_version.py b/src/python/grpcio_admin/grpc_version.py index 4a25fb631dd..f9ca61c6172 100644 --- a/src/python/grpcio_admin/grpc_version.py +++ b/src/python/grpcio_admin/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_admin/grpc_version.py.template`!!! -VERSION = '1.62.0.dev0' +VERSION = '1.63.0.dev0' diff --git a/src/python/grpcio_channelz/grpc_version.py b/src/python/grpcio_channelz/grpc_version.py index 3f206d308a4..59c591c0543 100644 --- a/src/python/grpcio_channelz/grpc_version.py +++ b/src/python/grpcio_channelz/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_channelz/grpc_version.py.template`!!! -VERSION = '1.62.0.dev0' +VERSION = '1.63.0.dev0' diff --git a/src/python/grpcio_csds/grpc_version.py b/src/python/grpcio_csds/grpc_version.py index f3bfc9532eb..5bd9b77ddee 100644 --- a/src/python/grpcio_csds/grpc_version.py +++ b/src/python/grpcio_csds/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_csds/grpc_version.py.template`!!! -VERSION = '1.62.0.dev0' +VERSION = '1.63.0.dev0' diff --git a/src/python/grpcio_health_checking/grpc_version.py b/src/python/grpcio_health_checking/grpc_version.py index 66bdbae0532..26e6969b6be 100644 --- a/src/python/grpcio_health_checking/grpc_version.py +++ b/src/python/grpcio_health_checking/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_health_checking/grpc_version.py.template`!!! -VERSION = '1.62.0.dev0' +VERSION = '1.63.0.dev0' diff --git a/src/python/grpcio_observability/grpc_version.py b/src/python/grpcio_observability/grpc_version.py index 400811356af..318362a9ac0 100644 --- a/src/python/grpcio_observability/grpc_version.py +++ b/src/python/grpcio_observability/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_observability/grpc_version.py.template`!!! -VERSION = '1.62.0.dev0' +VERSION = '1.63.0.dev0' diff --git a/src/python/grpcio_reflection/grpc_version.py b/src/python/grpcio_reflection/grpc_version.py index be7becf78d5..8916545f86c 100644 --- a/src/python/grpcio_reflection/grpc_version.py +++ b/src/python/grpcio_reflection/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_reflection/grpc_version.py.template`!!! -VERSION = '1.62.0.dev0' +VERSION = '1.63.0.dev0' diff --git a/src/python/grpcio_status/grpc_version.py b/src/python/grpcio_status/grpc_version.py index 287f141a033..33f6c90a611 100644 --- a/src/python/grpcio_status/grpc_version.py +++ b/src/python/grpcio_status/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_status/grpc_version.py.template`!!! -VERSION = '1.62.0.dev0' +VERSION = '1.63.0.dev0' diff --git a/src/python/grpcio_testing/grpc_version.py b/src/python/grpcio_testing/grpc_version.py index ac5db88749b..155aba78b27 100644 --- a/src/python/grpcio_testing/grpc_version.py +++ b/src/python/grpcio_testing/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_testing/grpc_version.py.template`!!! -VERSION = '1.62.0.dev0' +VERSION = '1.63.0.dev0' diff --git a/src/python/grpcio_tests/grpc_version.py b/src/python/grpcio_tests/grpc_version.py index ad10467cbfb..65205cccaf6 100644 --- a/src/python/grpcio_tests/grpc_version.py +++ b/src/python/grpcio_tests/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_tests/grpc_version.py.template`!!! -VERSION = '1.62.0.dev0' +VERSION = '1.63.0.dev0' diff --git a/src/ruby/lib/grpc/version.rb b/src/ruby/lib/grpc/version.rb index 72fffd62e21..6b551866e28 100644 --- a/src/ruby/lib/grpc/version.rb +++ b/src/ruby/lib/grpc/version.rb @@ -14,5 +14,5 @@ # GRPC contains the General RPC module. module GRPC - VERSION = '1.62.0.dev' + VERSION = '1.63.0.dev' end diff --git a/src/ruby/nativedebug/version.rb b/src/ruby/nativedebug/version.rb index 048eef0ad1d..ffcd151a8db 100644 --- a/src/ruby/nativedebug/version.rb +++ b/src/ruby/nativedebug/version.rb @@ -14,6 +14,6 @@ module GRPC module NativeDebug - VERSION = '1.62.0.dev' + VERSION = '1.63.0.dev' end end diff --git a/src/ruby/tools/version.rb b/src/ruby/tools/version.rb index df15828df03..3152ec80a41 100644 --- a/src/ruby/tools/version.rb +++ b/src/ruby/tools/version.rb @@ -14,6 +14,6 @@ module GRPC module Tools - VERSION = '1.62.0.dev' + VERSION = '1.63.0.dev' end end diff --git a/tools/distrib/python/grpc_version.py b/tools/distrib/python/grpc_version.py index 2e67d20c09c..173e7231f45 100644 --- a/tools/distrib/python/grpc_version.py +++ b/tools/distrib/python/grpc_version.py @@ -14,5 +14,5 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION = '1.62.0.dev0' +VERSION = '1.63.0.dev0' PROTOBUF_VERSION = '3.25.1' diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index 61ac8012bd5..bffb456c0aa 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -14,5 +14,5 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION = '1.62.0.dev0' +VERSION = '1.63.0.dev0' PROTOBUF_VERSION = '3.25.1' diff --git a/tools/distrib/python/xds_protos/grpc_version.py b/tools/distrib/python/xds_protos/grpc_version.py index 3b3c9836035..b47e5ab8cf0 100644 --- a/tools/distrib/python/xds_protos/grpc_version.py +++ b/tools/distrib/python/xds_protos/grpc_version.py @@ -14,5 +14,5 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION = '1.62.0.dev0' +VERSION = '1.63.0.dev0' PROTOBUF_VERSION = '3.25.1' diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 6504112d5d8..81f77ceb264 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.62.0-dev +PROJECT_NUMBER = 1.63.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 413782870eb..27f47e275a9 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.62.0-dev +PROJECT_NUMBER = 1.63.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.objc b/tools/doxygen/Doxyfile.objc index 17c0826b946..8aab6a36ae9 100644 --- a/tools/doxygen/Doxyfile.objc +++ b/tools/doxygen/Doxyfile.objc @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC Objective-C" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.62.0-dev +PROJECT_NUMBER = 1.63.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.objc.internal b/tools/doxygen/Doxyfile.objc.internal index f5b347f4c63..905cdd9fb60 100644 --- a/tools/doxygen/Doxyfile.objc.internal +++ b/tools/doxygen/Doxyfile.objc.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC Objective-C" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.62.0-dev +PROJECT_NUMBER = 1.63.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.php b/tools/doxygen/Doxyfile.php index 1a5c08b254b..5681c345cd0 100644 --- a/tools/doxygen/Doxyfile.php +++ b/tools/doxygen/Doxyfile.php @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC PHP" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.62.0-dev +PROJECT_NUMBER = 1.63.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a From 527557d03daf64c7bca5987dd167a0b02a3d2a01 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Tue, 13 Feb 2024 16:12:57 -0800 Subject: [PATCH 15/37] [Deps] Upgrade abseil podsepc to 1.20240116.1 (#35893) Closes #35893 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/35893 from veblush:absl-osx e01dd83ee23fe0cc2ee0ded2232dc60b2f92a58d PiperOrigin-RevId: 606780054 --- gRPC-C++.podspec | 2 +- gRPC-Core.podspec | 2 +- templates/gRPC-C++.podspec.template | 2 +- templates/gRPC-Core.podspec.template | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index bdfe95324ae..46b13b49118 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -225,7 +225,7 @@ Pod::Spec.new do |s| ss.dependency "#{s.name}/Privacy", version ss.dependency "#{s.name}/Interface", version ss.dependency 'gRPC-Core', version - abseil_version = '1.20240116.0' + abseil_version = '1.20240116.1' ss.dependency 'abseil/algorithm/container', abseil_version ss.dependency 'abseil/base/base', abseil_version ss.dependency 'abseil/base/config', abseil_version diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index e7989599bb6..3e6029dc38e 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -46,7 +46,7 @@ Pod::Spec.new do |s| s.requires_arc = false name = 'grpc' - abseil_version = '1.20240116.0' + abseil_version = '1.20240116.1' # When creating a dynamic framework, name it grpc.framework instead of gRPC-Core.framework. # This lets users write their includes like `#include ` as opposed to `#include diff --git a/templates/gRPC-C++.podspec.template b/templates/gRPC-C++.podspec.template index 2a4b559d705..b4c24348bf1 100644 --- a/templates/gRPC-C++.podspec.template +++ b/templates/gRPC-C++.podspec.template @@ -183,7 +183,7 @@ ss.dependency "#{s.name}/Privacy", version ss.dependency "#{s.name}/Interface", version ss.dependency 'gRPC-Core', version - abseil_version = '1.20240116.0' + abseil_version = '1.20240116.1' % for abseil_spec in grpcpp_abseil_specs: ss.dependency '${abseil_spec}', abseil_version % endfor diff --git a/templates/gRPC-Core.podspec.template b/templates/gRPC-Core.podspec.template index 2a20a9c5a94..e1ddf100bfd 100644 --- a/templates/gRPC-Core.podspec.template +++ b/templates/gRPC-Core.podspec.template @@ -122,7 +122,7 @@ s.requires_arc = false name = 'grpc' - abseil_version = '1.20240116.0' + abseil_version = '1.20240116.1' # When creating a dynamic framework, name it grpc.framework instead of gRPC-Core.framework. # This lets users write their includes like `#include ` as opposed to `#include From b2b6af691ed7bb923d8288a05167526d946ea1eb Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Wed, 14 Feb 2024 10:03:44 -0800 Subject: [PATCH 16/37] Treat empty response as non-200 for IPv6 query in c2p resolver PiperOrigin-RevId: 607022595 --- src/core/resolver/google_c2p/google_c2p_resolver.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/core/resolver/google_c2p/google_c2p_resolver.cc b/src/core/resolver/google_c2p/google_c2p_resolver.cc index 657ac27e35d..78bc88bf25c 100644 --- a/src/core/resolver/google_c2p/google_c2p_resolver.cc +++ b/src/core/resolver/google_c2p/google_c2p_resolver.cc @@ -173,7 +173,11 @@ void GoogleCloud2ProdResolver::StartLocked() { absl::StatusOr result) mutable { resolver->work_serializer_->Run( [resolver, result = std::move(result)]() { - resolver->IPv6QueryDone(result.ok()); + // Check that the payload is non-empty in order to work around + // the fact that there are buggy implementations of metadata + // servers in the wild, which can in some cases return 200 + // plus an empty result when they should have returned 404. + resolver->IPv6QueryDone(result.ok() && !result->empty()); }, DEBUG_LOCATION); }, From b3b64dc715e0d946c172c78be338ca73dbfaa90c Mon Sep 17 00:00:00 2001 From: Gregory Cooke Date: Wed, 14 Feb 2024 11:50:01 -0800 Subject: [PATCH 17/37] [Security] Handshaker status return (#34864) Address the first (easy) half of https://github.com/grpc/grpc/issues/32598 Closes #34864 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/34864 from gtcooke94:HandshakerStatusReturn 8d33012b2ddb8af613902187ccbf1109e21999b8 PiperOrigin-RevId: 607058614 --- .../security/transport/security_handshaker.cc | 27 ++++++++++++------- .../security/transport/security_handshaker.h | 6 +++-- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/core/lib/security/transport/security_handshaker.cc b/src/core/lib/security/transport/security_handshaker.cc index 885b35fc9d2..8014ce03063 100644 --- a/src/core/lib/security/transport/security_handshaker.cc +++ b/src/core/lib/security/transport/security_handshaker.cc @@ -27,6 +27,7 @@ #include #include #include +#include #include "absl/base/attributes.h" #include "absl/status/status.h" @@ -585,25 +586,25 @@ void SecurityHandshaker::DoHandshake(grpc_tcp_server_acceptor* /*acceptor*/, class FailHandshaker : public Handshaker { public: + explicit FailHandshaker(absl::Status status) : status_(std::move(status)) {} const char* name() const override { return "security_fail"; } void Shutdown(grpc_error_handle /*why*/) override {} void DoHandshake(grpc_tcp_server_acceptor* /*acceptor*/, grpc_closure* on_handshake_done, HandshakerArgs* args) override { - grpc_error_handle error = - GRPC_ERROR_CREATE("Failed to create security handshaker"); - grpc_endpoint_shutdown(args->endpoint, error); + grpc_endpoint_shutdown(args->endpoint, status_); grpc_endpoint_destroy(args->endpoint); args->endpoint = nullptr; args->args = ChannelArgs(); grpc_slice_buffer_destroy(args->read_buffer); gpr_free(args->read_buffer); args->read_buffer = nullptr; - ExecCtx::Run(DEBUG_LOCATION, on_handshake_done, error); + ExecCtx::Run(DEBUG_LOCATION, on_handshake_done, status_); } private: ~FailHandshaker() override = default; + absl::Status status_; }; // @@ -652,14 +653,22 @@ class ServerSecurityHandshakerFactory : public HandshakerFactory { // RefCountedPtr SecurityHandshakerCreate( - tsi_handshaker* handshaker, grpc_security_connector* connector, - const ChannelArgs& args) { + absl::StatusOr handshaker, + grpc_security_connector* connector, const ChannelArgs& args) { // If no TSI handshaker was created, return a handshaker that always fails. // Otherwise, return a real security handshaker. - if (handshaker == nullptr) { - return MakeRefCounted(); + if (!handshaker.ok()) { + return MakeRefCounted( + absl::Status(handshaker.status().code(), + absl::StrCat("Failed to create security handshaker: ", + handshaker.status().message()))); + } else if (*handshaker == nullptr) { + // TODO(gtcooke94) Once all TSI impls are updated to pass StatusOr<> instead + // of null, we should change this to use absl::InternalError(). + return MakeRefCounted( + absl::UnknownError("Failed to create security handshaker.")); } else { - return MakeRefCounted(handshaker, connector, args); + return MakeRefCounted(*handshaker, connector, args); } } diff --git a/src/core/lib/security/transport/security_handshaker.h b/src/core/lib/security/transport/security_handshaker.h index f6f5d42a462..725584f37b3 100644 --- a/src/core/lib/security/transport/security_handshaker.h +++ b/src/core/lib/security/transport/security_handshaker.h @@ -21,6 +21,8 @@ #include +#include "absl/status/statusor.h" + #include #include "src/core/lib/channel/channel_args.h" @@ -34,8 +36,8 @@ namespace grpc_core { /// Creates a security handshaker using \a handshaker. RefCountedPtr SecurityHandshakerCreate( - tsi_handshaker* handshaker, grpc_security_connector* connector, - const ChannelArgs& args); + absl::StatusOr handshaker, + grpc_security_connector* connector, const ChannelArgs& args); /// Registers security handshaker factories. void SecurityRegisterHandshakerFactories(CoreConfiguration::Builder*); From a58f83c947be52d4b622aa0d32435323b81a8e5e Mon Sep 17 00:00:00 2001 From: youyuanwu <48816116+youyuanwu@users.noreply.github.com> Date: Wed, 14 Feb 2024 13:14:04 -0800 Subject: [PATCH 18/37] [EventEngine] Support AF_UNIX for windows (#34801) #22285 Unix domain socket has been added to windows: [AF_UNIX comes to Windows](https://devblogs.microsoft.com/commandline/af_unix-comes-to-windows/) Golang net pkg has adopted this long ago: https://go-review.googlesource.com/c/go/+/125456 https://go-review.googlesource.com/c/sys/+/132555 grpc-go already support this. AF_UNIX on windows is seamlessly integrated with winsock API. The modification needed are: * Set the right address family AF_UNIX depending on the address config, instead of using AF_INET6 all the time. * Ignore socket options for tcp. Closes #34801 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/34801 from youyuanwu:dev 3d9b5c097bc6394f113be4e55173a4e8d4d85e9d PiperOrigin-RevId: 607083164 --- .../binder/client/binder_connector.cc | 7 ++ src/core/lib/address_utils/parse_address.cc | 9 ++- src/core/lib/address_utils/sockaddr_utils.cc | 9 ++- .../posix_engine/tcp_socket_utils.cc | 7 ++ src/core/lib/event_engine/tcp_socket_utils.cc | 7 ++ .../lib/event_engine/windows/win_socket.cc | 4 ++ .../lib/event_engine/windows/win_socket.h | 3 + .../event_engine/windows/windows_engine.cc | 22 +++++- .../event_engine/windows/windows_listener.cc | 58 ++++++++++++++-- .../event_engine/windows/windows_listener.h | 14 ++++ src/core/lib/iomgr/port.h | 3 + src/core/lib/iomgr/tcp_client_windows.cc | 29 ++++++-- src/core/lib/iomgr/tcp_server_windows.cc | 69 ++++++++++++++++--- src/core/lib/iomgr/unix_sockets_posix.cc | 13 ++++ src/core/resolver/binder/binder_resolver.cc | 7 ++ test/core/address_utils/parse_address_test.cc | 9 ++- .../core/address_utils/sockaddr_utils_test.cc | 9 ++- .../resolvers/binder_resolver_test.cc | 9 ++- test/core/end2end/end2end_test_suites.cc | 40 ++++++++--- test/core/end2end/tests/default_host.cc | 3 +- .../event_engine/tcp_socket_utils_test.cc | 9 ++- tools/run_tests/run_tests.py | 4 ++ 22 files changed, 306 insertions(+), 38 deletions(-) diff --git a/src/core/ext/transport/binder/client/binder_connector.cc b/src/core/ext/transport/binder/client/binder_connector.cc index c948e372ad5..0c3db89927c 100644 --- a/src/core/ext/transport/binder/client/binder_connector.cc +++ b/src/core/ext/transport/binder/client/binder_connector.cc @@ -20,7 +20,14 @@ #include "src/core/lib/iomgr/port.h" #ifdef GRPC_HAVE_UNIX_SOCKET +#ifdef GPR_WINDOWS +// clang-format off +#include +#include +// clang-format on +#else #include +#endif // GPR_WINDOWS #endif #include diff --git a/src/core/lib/address_utils/parse_address.cc b/src/core/lib/address_utils/parse_address.cc index deb2b079723..4806d170988 100644 --- a/src/core/lib/address_utils/parse_address.cc +++ b/src/core/lib/address_utils/parse_address.cc @@ -30,8 +30,15 @@ #include #include #ifdef GRPC_HAVE_UNIX_SOCKET +#ifdef GPR_WINDOWS +// clang-format off +#include +#include +// clang-format on +#else #include -#endif +#endif // GPR_WINDOWS +#endif // GRPC_HAVE_UNIX_SOCKET #include #include "absl/status/status.h" diff --git a/src/core/lib/address_utils/sockaddr_utils.cc b/src/core/lib/address_utils/sockaddr_utils.cc index a1442afe5da..c82efe927a1 100644 --- a/src/core/lib/address_utils/sockaddr_utils.cc +++ b/src/core/lib/address_utils/sockaddr_utils.cc @@ -44,8 +44,15 @@ #include "src/core/lib/uri/uri_parser.h" #ifdef GRPC_HAVE_UNIX_SOCKET +#ifdef GPR_WINDOWS +// clang-format off +#include +#include +// clang-format on +#else #include -#endif +#endif // GPR_WINDOWS +#endif // GRPC_HAVE_UNIX_SOCKET #ifdef GRPC_HAVE_UNIX_SOCKET static absl::StatusOr grpc_sockaddr_to_uri_unix_if_possible( diff --git a/src/core/lib/event_engine/posix_engine/tcp_socket_utils.cc b/src/core/lib/event_engine/posix_engine/tcp_socket_utils.cc index 76e1b0a57e1..ab98713b62a 100644 --- a/src/core/lib/event_engine/posix_engine/tcp_socket_utils.cc +++ b/src/core/lib/event_engine/posix_engine/tcp_socket_utils.cc @@ -58,8 +58,15 @@ #include "src/core/lib/gprpp/strerror.h" #ifdef GRPC_HAVE_UNIX_SOCKET +#ifdef GPR_WINDOWS +// clang-format off +#include +#include +// clang-format on +#else #include // IWYU pragma: keep #include +#endif // GPR_WINDOWS #endif namespace grpc_event_engine { diff --git a/src/core/lib/event_engine/tcp_socket_utils.cc b/src/core/lib/event_engine/tcp_socket_utils.cc index a5fa15223d1..c3bd35fc905 100644 --- a/src/core/lib/event_engine/tcp_socket_utils.cc +++ b/src/core/lib/event_engine/tcp_socket_utils.cc @@ -32,8 +32,15 @@ #endif // GRPC_POSIX_SOCKET_UTILS_COMMON #ifdef GRPC_HAVE_UNIX_SOCKET +#ifdef GPR_WINDOWS +// clang-format off +#include +#include +// clang-format on +#else #include // IWYU pragma: keep #include +#endif // GPR_WINDOWS #endif #ifdef GRPC_HAVE_VSOCK diff --git a/src/core/lib/event_engine/windows/win_socket.cc b/src/core/lib/event_engine/windows/win_socket.cc index 22ee56650d2..da919e2ba7c 100644 --- a/src/core/lib/event_engine/windows/win_socket.cc +++ b/src/core/lib/event_engine/windows/win_socket.cc @@ -208,6 +208,10 @@ static grpc_error_handle enable_socket_low_latency(SOCKET sock) { } // namespace +absl::Status SetSocketNonBlock(SOCKET sock) { + return grpc_tcp_set_non_block(sock); +} + absl::Status PrepareSocket(SOCKET sock) { absl::Status err; err = grpc_tcp_set_non_block(sock); diff --git a/src/core/lib/event_engine/windows/win_socket.h b/src/core/lib/event_engine/windows/win_socket.h index bc8e8061c81..c6d52f41def 100644 --- a/src/core/lib/event_engine/windows/win_socket.h +++ b/src/core/lib/event_engine/windows/win_socket.h @@ -133,6 +133,9 @@ class WinSocket { // Attempt to configure default socket settings absl::Status PrepareSocket(SOCKET sock); +// Set non block option for socket. +absl::Status SetSocketNonBlock(SOCKET sock); + } // namespace experimental } // namespace grpc_event_engine diff --git a/src/core/lib/event_engine/windows/windows_engine.cc b/src/core/lib/event_engine/windows/windows_engine.cc index 5503f3fc85b..f4431ae4ef4 100644 --- a/src/core/lib/event_engine/windows/windows_engine.cc +++ b/src/core/lib/event_engine/windows/windows_engine.cc @@ -306,7 +306,10 @@ EventEngine::ConnectionHandle WindowsEventEngine::Connect( if (ResolvedAddressToV4Mapped(addr, &addr6_v4mapped)) { address = addr6_v4mapped; } - SOCKET sock = WSASocket(AF_INET6, SOCK_STREAM, IPPROTO_TCP, nullptr, 0, + const int addr_family = + (address.address()->sa_family == AF_UNIX) ? AF_UNIX : AF_INET6; + const int protocol = addr_family == AF_UNIX ? 0 : IPPROTO_TCP; + SOCKET sock = WSASocket(addr_family, SOCK_STREAM, protocol, nullptr, 0, IOCP::GetDefaultSocketFlags()); if (sock == INVALID_SOCKET) { Run([on_connect = std::move(on_connect), @@ -315,7 +318,11 @@ EventEngine::ConnectionHandle WindowsEventEngine::Connect( }); return EventEngine::ConnectionHandle::kInvalid; } - status = PrepareSocket(sock); + if (addr_family == AF_UNIX) { + status = SetSocketNonBlock(sock); + } else { + status = PrepareSocket(sock); + } if (!status.ok()) { Run([on_connect = std::move(on_connect), status]() mutable { on_connect(status); @@ -340,7 +347,16 @@ EventEngine::ConnectionHandle WindowsEventEngine::Connect( return EventEngine::ConnectionHandle::kInvalid; } // bind the local address - auto local_address = ResolvedAddressMakeWild6(0); + ResolvedAddress local_address; + if (addr_family == AF_UNIX) { + // For ConnectEx() to work for AF_UNIX, the sock needs to be bound to + // the local address of an unnamed socket. + sockaddr addr = {}; + addr.sa_family = AF_UNIX; + local_address = ResolvedAddress(&addr, sizeof(addr)); + } else { + local_address = ResolvedAddressMakeWild6(0); + } istatus = bind(sock, local_address.address(), local_address.size()); if (istatus != 0) { Run([on_connect = std::move(on_connect), diff --git a/src/core/lib/event_engine/windows/windows_listener.cc b/src/core/lib/event_engine/windows/windows_listener.cc index 71116a63326..d74cfc0e8bc 100644 --- a/src/core/lib/event_engine/windows/windows_listener.cc +++ b/src/core/lib/event_engine/windows/windows_listener.cc @@ -27,6 +27,7 @@ #include "src/core/lib/gprpp/crash.h" #include "src/core/lib/gprpp/sync.h" #include "src/core/lib/iomgr/error.h" +#include "src/core/lib/iomgr/port.h" namespace grpc_event_engine { namespace experimental { @@ -67,11 +68,39 @@ void WindowsEventEngineListener::SinglePortSocketListener:: // ---- SinglePortSocketListener ---- +// TODO(hork): This may be refactored to share with posix engine. +void UnlinkIfUnixDomainSocket( + const EventEngine::ResolvedAddress& resolved_addr) { +#ifdef GRPC_HAVE_UNIX_SOCKET + if (resolved_addr.address()->sa_family != AF_UNIX) { + return; + } + struct sockaddr_un* un = reinterpret_cast( + const_cast(resolved_addr.address())); + // There is nothing to unlink for an abstract unix socket. + if (un->sun_path[0] == '\0' && un->sun_path[1] != '\0') { + return; + } + // For windows we need to remove the file instead of unlink. + DWORD attr = ::GetFileAttributesA(un->sun_path); + if (attr == INVALID_FILE_ATTRIBUTES) { + return; + } + if (attr & FILE_ATTRIBUTE_DIRECTORY || attr & FILE_ATTRIBUTE_READONLY) { + return; + } + ::DeleteFileA(un->sun_path); +#else + (void)resolved_addr; +#endif +} + WindowsEventEngineListener::SinglePortSocketListener:: ~SinglePortSocketListener() { grpc_core::MutexLock lock(&io_state_->mu); io_state_->listener_socket->Shutdown(DEBUG_LOCATION, "~SinglePortSocketListener"); + UnlinkIfUnixDomainSocket(listener_sockname()); GRPC_EVENT_ENGINE_TRACE("~SinglePortSocketListener::%p", this); } @@ -109,7 +138,11 @@ absl::Status WindowsEventEngineListener::SinglePortSocketListener::Start() { absl::Status WindowsEventEngineListener::SinglePortSocketListener::StartLocked() { - SOCKET accept_socket = WSASocket(AF_INET6, SOCK_STREAM, IPPROTO_TCP, NULL, 0, + const EventEngine::ResolvedAddress addr = listener_sockname(); + const int addr_family = + (addr.address()->sa_family == AF_UNIX) ? AF_UNIX : AF_INET6; + const int protocol = addr_family == AF_UNIX ? 0 : IPPROTO_TCP; + SOCKET accept_socket = WSASocket(addr_family, SOCK_STREAM, protocol, NULL, 0, IOCP::GetDefaultSocketFlags()); if (accept_socket == INVALID_SOCKET) { return GRPC_WSA_ERROR(WSAGetLastError(), "WSASocket"); @@ -118,11 +151,17 @@ WindowsEventEngineListener::SinglePortSocketListener::StartLocked() { if (accept_socket != INVALID_SOCKET) closesocket(accept_socket); return error; }; - auto error = PrepareSocket(accept_socket); + absl::Status error; + if (addr_family == AF_UNIX) { + error = SetSocketNonBlock(accept_socket); + } else { + error = PrepareSocket(accept_socket); + } if (!error.ok()) return fail(error); // Start the "accept" asynchronously. io_state_->listener_socket->NotifyOnRead(&io_state_->on_accept_cb); - DWORD addrlen = sizeof(sockaddr_in6) + 16; + DWORD addrlen = + sizeof(addresses_) / 2; // half of the buffer is for remote addr. DWORD bytes_received = 0; int success = AcceptEx(io_state_->listener_socket->raw_socket(), accept_socket, @@ -238,8 +277,14 @@ WindowsEventEngineListener::SinglePortSocketListener::PrepareListenerSocket( if (sock != INVALID_SOCKET) closesocket(sock); return error; }; - auto error = PrepareSocket(sock); + absl::Status error; + if (addr.address()->sa_family == AF_UNIX) { + error = SetSocketNonBlock(sock); + } else { + error = PrepareSocket(sock); + } if (!error.ok()) return fail(error); + UnlinkIfUnixDomainSocket(addr); if (bind(sock, addr.address(), addr.size()) == SOCKET_ERROR) { return fail(GRPC_WSA_ERROR(WSAGetLastError(), "bind")); } @@ -313,7 +358,10 @@ absl::StatusOr WindowsEventEngineListener::Bind( out_addr = ResolvedAddressMakeWild6(out_port); } // open the socket - SOCKET sock = WSASocket(AF_INET6, SOCK_STREAM, IPPROTO_TCP, nullptr, 0, + const int addr_family = + (out_addr.address()->sa_family == AF_UNIX) ? AF_UNIX : AF_INET6; + const int protocol = addr_family == AF_UNIX ? 0 : IPPROTO_TCP; + SOCKET sock = WSASocket(addr_family, SOCK_STREAM, protocol, nullptr, 0, IOCP::GetDefaultSocketFlags()); if (sock == INVALID_SOCKET) { auto error = GRPC_WSA_ERROR(WSAGetLastError(), "WSASocket"); diff --git a/src/core/lib/event_engine/windows/windows_listener.h b/src/core/lib/event_engine/windows/windows_listener.h index a84238c3b7c..2773f3afbf3 100644 --- a/src/core/lib/event_engine/windows/windows_listener.h +++ b/src/core/lib/event_engine/windows/windows_listener.h @@ -30,6 +30,14 @@ #include "src/core/lib/event_engine/thread_pool/thread_pool.h" #include "src/core/lib/event_engine/windows/iocp.h" #include "src/core/lib/gprpp/sync.h" +#include "src/core/lib/iomgr/port.h" + +#ifdef GRPC_HAVE_UNIX_SOCKET +// clang-format off +#include +#include +// clang-format on +#endif namespace grpc_event_engine { namespace experimental { @@ -120,9 +128,15 @@ class WindowsEventEngineListener : public EventEngine::Listener { // The cached AcceptEx for that port. LPFN_ACCEPTEX AcceptEx; + // Buffer to hold the local and remote address. // This seemingly magic number comes from AcceptEx's documentation. each // address buffer needs to have at least 16 more bytes at their end. +#ifdef GRPC_HAVE_UNIX_SOCKET + // unix addr is larger than ip addr. + uint8_t addresses_[(sizeof(sockaddr_un) + 16) * 2] = {}; +#else uint8_t addresses_[(sizeof(sockaddr_in6) + 16) * 2] = {}; +#endif // The parent listener WindowsEventEngineListener* listener_; // shared state for asynchronous cleanup of overlapped operations diff --git a/src/core/lib/iomgr/port.h b/src/core/lib/iomgr/port.h index 3385408309b..f4102a524a2 100644 --- a/src/core/lib/iomgr/port.h +++ b/src/core/lib/iomgr/port.h @@ -30,6 +30,9 @@ #endif #if defined(GPR_WINDOWS) #define GRPC_WINSOCK_SOCKET 1 +#ifndef __MINGW32__ +#define GRPC_HAVE_UNIX_SOCKET 1 +#endif // __MINGW32__ #define GRPC_WINDOWS_SOCKETUTILS 1 #define GRPC_WINDOWS_SOCKET_ARES_EV_DRIVER 1 #elif defined(GPR_ANDROID) diff --git a/src/core/lib/iomgr/tcp_client_windows.cc b/src/core/lib/iomgr/tcp_client_windows.cc index 7a4d7ca3103..637c6e3cb55 100644 --- a/src/core/lib/iomgr/tcp_client_windows.cc +++ b/src/core/lib/iomgr/tcp_client_windows.cc @@ -145,6 +145,8 @@ static int64_t tcp_connect(grpc_closure* on_done, grpc_endpoint** endpoint, grpc_error_handle error; async_connect* ac = NULL; absl::StatusOr addr_uri; + int addr_family; + int protocol; addr_uri = grpc_sockaddr_to_uri(addr); if (!addr_uri.ok()) { @@ -159,14 +161,25 @@ static int64_t tcp_connect(grpc_closure* on_done, grpc_endpoint** endpoint, addr = &addr6_v4mapped; } - sock = WSASocket(AF_INET6, SOCK_STREAM, IPPROTO_TCP, NULL, 0, + // extract family + addr_family = + (grpc_sockaddr_get_family(addr) == AF_UNIX) ? AF_UNIX : AF_INET6; + protocol = addr_family == AF_UNIX ? 0 : IPPROTO_TCP; + + sock = WSASocket(addr_family, SOCK_STREAM, protocol, NULL, 0, grpc_get_default_wsa_socket_flags()); if (sock == INVALID_SOCKET) { error = GRPC_WSA_ERROR(WSAGetLastError(), "WSASocket"); goto failure; } - error = grpc_tcp_prepare_socket(sock); + if (addr_family == AF_UNIX) { + // tcp settings for af_unix are skipped. + error = grpc_tcp_set_non_block(sock); + } else { + error = grpc_tcp_prepare_socket(sock); + } + if (!error.ok()) { goto failure; } @@ -183,7 +196,15 @@ static int64_t tcp_connect(grpc_closure* on_done, grpc_endpoint** endpoint, goto failure; } - grpc_sockaddr_make_wildcard6(0, &local_address); + if (addr_family == AF_UNIX) { + // For ConnectEx() to work for AF_UNIX, the sock needs to be bound to + // the local address of an unnamed socket. + local_address = {}; + ((grpc_sockaddr*)local_address.addr)->sa_family = AF_UNIX; + local_address.len = sizeof(grpc_sockaddr); + } else { + grpc_sockaddr_make_wildcard6(0, &local_address); + } status = bind(sock, (grpc_sockaddr*)&local_address.addr, (int)local_address.len); @@ -196,7 +217,6 @@ static int64_t tcp_connect(grpc_closure* on_done, grpc_endpoint** endpoint, info = &socket->write_info; success = ConnectEx(sock, (grpc_sockaddr*)&addr->addr, (int)addr->len, NULL, 0, NULL, &info->overlapped); - // It wouldn't be unusual to get a success immediately. But we'll still get // an IOCP notification, so let's ignore it. if (!success) { @@ -206,7 +226,6 @@ static int64_t tcp_connect(grpc_closure* on_done, grpc_endpoint** endpoint, goto failure; } } - ac = new async_connect(); ac->on_done = on_done; ac->socket = socket; diff --git a/src/core/lib/iomgr/tcp_server_windows.cc b/src/core/lib/iomgr/tcp_server_windows.cc index d7775a1e267..9de6cb0f268 100644 --- a/src/core/lib/iomgr/tcp_server_windows.cc +++ b/src/core/lib/iomgr/tcp_server_windows.cc @@ -78,13 +78,21 @@ using ::grpc_event_engine::experimental::WindowsEventEngineListener; // one listening port typedef struct grpc_tcp_listener grpc_tcp_listener; struct grpc_tcp_listener { + // Buffer to hold the local and remote address. // This seemingly magic number comes from AcceptEx's documentation. each // address buffer needs to have at least 16 more bytes at their end. +#ifdef GRPC_HAVE_UNIX_SOCKET + // unix addr is larger than ip addr. + uint8_t addresses[(sizeof(sockaddr_un) + 16) * 2] = {}; +#else uint8_t addresses[(sizeof(grpc_sockaddr_in6) + 16) * 2]; +#endif // GRPC_HAVE_UNIX_SOCKET // This will hold the socket for the next accept. SOCKET new_socket; // The listener winsocket. grpc_winsocket* socket; + // address of listener + grpc_resolved_address resolved_addr; // The actual TCP port number. int port; unsigned port_index; @@ -125,6 +133,35 @@ struct grpc_tcp_server { WindowsEventEngineListener* ee_listener; }; +// TODO(hork): This may be refactored to share with posix engine and event +// engine. +void unlink_if_unix_domain_socket(const grpc_resolved_address* resolved_addr) { +#ifdef GRPC_HAVE_UNIX_SOCKET + const grpc_sockaddr* addr = + reinterpret_cast(resolved_addr->addr); + if (addr->sa_family != AF_UNIX) { + return; + } + struct sockaddr_un* un = + reinterpret_cast(const_cast(addr)); + // There is nothing to unlink for an abstract unix socket. + if (un->sun_path[0] == '\0' && un->sun_path[1] != '\0') { + return; + } + // For windows we need to remove the file instead of unlink. + DWORD attr = ::GetFileAttributesA(un->sun_path); + if (attr == INVALID_FILE_ATTRIBUTES) { + return; + } + if (attr & FILE_ATTRIBUTE_DIRECTORY || attr & FILE_ATTRIBUTE_READONLY) { + return; + } + ::DeleteFileA(un->sun_path); +#else + (void)resolved_addr; +#endif +} + // Public function. Allocates the proper data structures to hold a // grpc_tcp_server. static grpc_error_handle tcp_server_create(grpc_closure* shutdown_complete, @@ -158,6 +195,7 @@ static void destroy_server(void* arg, grpc_error_handle /* error */) { s->head = sp->next; sp->next = NULL; grpc_winsocket_destroy(sp->socket); + unlink_if_unix_domain_socket(&sp->resolved_addr); gpr_free(sp); } gpr_mu_destroy(&s->mu); @@ -222,12 +260,15 @@ static grpc_error_handle prepare_socket(SOCKET sock, grpc_resolved_address sockname_temp; grpc_error_handle error; int sockname_temp_len; - - error = grpc_tcp_prepare_socket(sock); + if (grpc_sockaddr_get_family(addr) == AF_UNIX) { + error = grpc_tcp_set_non_block(sock); + } else { + error = grpc_tcp_prepare_socket(sock); + } if (!error.ok()) { goto failure; } - + unlink_if_unix_domain_socket(addr); if (bind(sock, (const grpc_sockaddr*)addr->addr, (int)addr->len) == SOCKET_ERROR) { error = GRPC_WSA_ERROR(WSAGetLastError(), "bind"); @@ -277,22 +318,28 @@ static void decrement_active_ports_and_notify_locked(grpc_tcp_listener* sp) { static grpc_error_handle start_accept_locked(grpc_tcp_listener* port) { SOCKET sock = INVALID_SOCKET; BOOL success; - DWORD addrlen = sizeof(grpc_sockaddr_in6) + 16; + const DWORD addrlen = sizeof(port->addresses) / 2; DWORD bytes_received = 0; grpc_error_handle error; if (port->shutting_down) { return absl::OkStatus(); } - - sock = WSASocket(AF_INET6, SOCK_STREAM, IPPROTO_TCP, NULL, 0, + const int addr_family = + grpc_sockaddr_get_family(&port->resolved_addr) == AF_UNIX ? AF_UNIX + : AF_INET6; + const int protocol = addr_family == AF_UNIX ? 0 : IPPROTO_TCP; + sock = WSASocket(addr_family, SOCK_STREAM, protocol, NULL, 0, grpc_get_default_wsa_socket_flags()); if (sock == INVALID_SOCKET) { error = GRPC_WSA_ERROR(WSAGetLastError(), "WSASocket"); goto failure; } - - error = grpc_tcp_prepare_socket(sock); + if (addr_family == AF_UNIX) { + error = grpc_tcp_set_non_block(sock); + } else { + error = grpc_tcp_prepare_socket(sock); + } if (!error.ok()) goto failure; // Start the "accept" asynchronously. @@ -463,6 +510,7 @@ static grpc_error_handle add_socket_to_server(grpc_tcp_server* s, SOCKET sock, sp->outstanding_calls = 0; sp->AcceptEx = AcceptEx; sp->new_socket = INVALID_SOCKET; + sp->resolved_addr = *addr; sp->port = port; sp->port_index = port_index; GRPC_CLOSURE_INIT(&sp->on_accept, on_accept, sp, grpc_schedule_on_exec_ctx); @@ -522,7 +570,10 @@ static grpc_error_handle tcp_server_add_port(grpc_tcp_server* s, addr = &wildcard; } - sock = WSASocket(AF_INET6, SOCK_STREAM, IPPROTO_TCP, NULL, 0, + const int addr_family = + grpc_sockaddr_get_family(addr) == AF_UNIX ? AF_UNIX : AF_INET6; + const int protocol = addr_family == AF_UNIX ? 0 : IPPROTO_TCP; + sock = WSASocket(addr_family, SOCK_STREAM, protocol, NULL, 0, grpc_get_default_wsa_socket_flags()); if (sock == INVALID_SOCKET) { error = GRPC_WSA_ERROR(WSAGetLastError(), "WSASocket"); diff --git a/src/core/lib/iomgr/unix_sockets_posix.cc b/src/core/lib/iomgr/unix_sockets_posix.cc index e182e6adf13..01daa2c90cf 100644 --- a/src/core/lib/iomgr/unix_sockets_posix.cc +++ b/src/core/lib/iomgr/unix_sockets_posix.cc @@ -24,7 +24,14 @@ #include #include #include +#ifdef GPR_WINDOWS +// clang-format off +#include +#include +// clang-format on +#else #include +#endif // GPR_WINDOWS #include "absl/strings/str_cat.h" @@ -39,7 +46,11 @@ #include "src/core/lib/transport/error_utils.h" void grpc_create_socketpair_if_unix(int sv[2]) { +#ifdef GPR_WINDOWS + grpc_core::Crash("AF_UNIX socket pairs are not supported on Windows"); +#else GPR_ASSERT(socketpair(AF_UNIX, SOCK_STREAM, 0, sv) == 0); +#endif } absl::StatusOr> @@ -86,10 +97,12 @@ void grpc_unlink_if_unix_domain_socket( return; } +#ifndef GPR_WINDOWS struct stat st; if (stat(un->sun_path, &st) == 0 && (st.st_mode & S_IFMT) == S_IFSOCK) { unlink(un->sun_path); } +#endif } #endif diff --git a/src/core/resolver/binder/binder_resolver.cc b/src/core/resolver/binder/binder_resolver.cc index 5f1916958a6..dfc8e15b2d8 100644 --- a/src/core/resolver/binder/binder_resolver.cc +++ b/src/core/resolver/binder/binder_resolver.cc @@ -24,8 +24,15 @@ #ifdef GRPC_HAVE_UNIX_SOCKET #include +#ifdef GPR_WINDOWS +// clang-format off +#include +#include +// clang-format on +#else #include #include +#endif // GPR_WINDOWS #include #include diff --git a/test/core/address_utils/parse_address_test.cc b/test/core/address_utils/parse_address_test.cc index 07b29c2678a..604d38aa300 100644 --- a/test/core/address_utils/parse_address_test.cc +++ b/test/core/address_utils/parse_address_test.cc @@ -18,8 +18,15 @@ #include "src/core/lib/address_utils/parse_address.h" #ifdef GRPC_HAVE_UNIX_SOCKET +#ifdef GPR_WINDOWS +// clang-format off +#include +#include +// clang-format on +#else #include -#endif +#endif // GPR_WINDOWS +#endif // GRPC_HAVE_UNIX_SOCKET #ifdef GRPC_HAVE_VSOCK #include diff --git a/test/core/address_utils/sockaddr_utils_test.cc b/test/core/address_utils/sockaddr_utils_test.cc index 1b4345c4b8a..74bb201cc1a 100644 --- a/test/core/address_utils/sockaddr_utils_test.cc +++ b/test/core/address_utils/sockaddr_utils_test.cc @@ -30,8 +30,15 @@ #include "src/core/lib/iomgr/port.h" #include "src/core/lib/iomgr/resolved_address.h" #ifdef GRPC_HAVE_UNIX_SOCKET +#ifdef GPR_WINDOWS +// clang-format off +#include +#include +// clang-format on +#else #include -#endif +#endif // GPR_WINDOWS +#endif // GRPC_HAVE_UNIX_SOCKET #include diff --git a/test/core/client_channel/resolvers/binder_resolver_test.cc b/test/core/client_channel/resolvers/binder_resolver_test.cc index 4abae7ee1a6..655f4c09cef 100644 --- a/test/core/client_channel/resolvers/binder_resolver_test.cc +++ b/test/core/client_channel/resolvers/binder_resolver_test.cc @@ -33,8 +33,15 @@ #ifdef GRPC_HAVE_UNIX_SOCKET +#ifdef GPR_WINDOWS +// clang-format off +#include +#include +// clang-format on +#else #include #include +#endif // GPR_WINDOWS #include #include @@ -201,7 +208,7 @@ TEST_F(BinderResolverTest, ValidCases) { "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~"); } -#endif +#endif // GRPC_HAVE_UNIX_SOCKET int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); diff --git a/test/core/end2end/end2end_test_suites.cc b/test/core/end2end/end2end_test_suites.cc index 6751b6e0dfc..625d1a8190a 100644 --- a/test/core/end2end/end2end_test_suites.cc +++ b/test/core/end2end/end2end_test_suites.cc @@ -568,6 +568,25 @@ class InsecureFixtureWithPipeForWakeupFd : public InsecureFixture { }; #endif +// Returns the temp directory to create uds in this test. +std::string GetTempDir() { +#ifdef GPR_WINDOWS + // Windows temp dir usually exceeds uds max paht length, + // so we create a short dir for this test. + // TODO: find a better solution. + std::string temp_dir = "C:/tmp/"; + if (CreateDirectoryA(temp_dir.c_str(), NULL) == 0 && + ERROR_ALREADY_EXISTS != GetLastError()) { + Crash(absl::StrCat("Could not create temp dir: ", temp_dir)); + } + return temp_dir; +#else + return "/tmp/"; +#endif // GPR_WINDOWS +} + +const std::string temp_dir = GetTempDir(); + std::vector DefaultConfigs() { return std::vector { #ifdef GRPC_POSIX_SOCKET @@ -662,9 +681,10 @@ std::vector DefaultConfigs() { gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); return std::make_unique( absl::StrFormat( - "unix:/tmp/grpc_fullstack_test.%%25.%d.%" PRId64 - ".%" PRId32 ".%" PRId64 ".%" PRId64, - getpid(), now.tv_sec, now.tv_nsec, + "unix:%s" + "grpc_fullstack_test.%%25.%d.%" PRId64 ".%" PRId32 + ".%" PRId64 ".%" PRId64, + temp_dir, getpid(), now.tv_sec, now.tv_nsec, unique.fetch_add(1, std::memory_order_relaxed), Rand()), UDS); }}, @@ -680,9 +700,10 @@ std::vector DefaultConfigs() { gpr_timespec now = gpr_now(GPR_CLOCK_REALTIME); return std::make_unique( absl::StrFormat( - "unix:/tmp/grpc_fullstack_test.%d.%" PRId64 ".%" PRId32 - ".%" PRId64 ".%" PRId64, - getpid(), now.tv_sec, now.tv_nsec, + "unix:%s" + "grpc_fullstack_test.%d.%" PRId64 ".%" PRId32 ".%" PRId64 + ".%" PRId64, + temp_dir, getpid(), now.tv_sec, now.tv_nsec, unique.fetch_add(1, std::memory_order_relaxed), Rand()), UDS); }}, @@ -924,9 +945,10 @@ std::vector DefaultConfigs() { [](const ChannelArgs&, const ChannelArgs&) { gpr_timespec now = gpr_now(GPR_CLOCK_REALTIME); return std::make_unique(absl::StrFormat( - "unix:/tmp/grpc_fullstack_test.%d.%" PRId64 ".%" PRId32 - ".%" PRId64 ".%" PRId64, - getpid(), now.tv_sec, now.tv_nsec, + "unix:%s" + "grpc_fullstack_test.%d.%" PRId64 ".%" PRId32 ".%" PRId64 + ".%" PRId64, + temp_dir, getpid(), now.tv_sec, now.tv_nsec, unique.fetch_add(1, std::memory_order_relaxed), Rand())); }}, #endif diff --git a/test/core/end2end/tests/default_host.cc b/test/core/end2end/tests/default_host.cc index 8bd53f3bb7a..770c15c4d61 100644 --- a/test/core/end2end/tests/default_host.cc +++ b/test/core/end2end/tests/default_host.cc @@ -65,7 +65,8 @@ CORE_END2END_TEST(CoreClientChannelTest, DefaultHost) { EXPECT_THAT(s.host(), AnyOf(StartsWith("localhost"), StartsWith("127.0.0.1"), StartsWith("[::1]"), StartsWith("grpc_fullstack_test."), - StartsWith("tmp%2Fgrpc_fullstack_test."))); + StartsWith("tmp%2Fgrpc_fullstack_test."), + StartsWith("C:%2Ftmp%2Fgrpc_fullstack_test."))); } EXPECT_FALSE(client_close.was_cancelled()); } diff --git a/test/core/event_engine/tcp_socket_utils_test.cc b/test/core/event_engine/tcp_socket_utils_test.cc index d3b114d8014..703ee19c5b7 100644 --- a/test/core/event_engine/tcp_socket_utils_test.cc +++ b/test/core/event_engine/tcp_socket_utils_test.cc @@ -35,8 +35,15 @@ #include #ifdef GRPC_HAVE_UNIX_SOCKET +#ifdef GPR_WINDOWS +// clang-format off +#include +#include +// clang-format on +#else #include -#endif +#endif // GPR_WINDOWS +#endif // GRPC_HAVE_UNIX_SOCKET #include "absl/status/status.h" #include "absl/status/statusor.h" diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index a072fdd17d0..5eb30557651 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -285,6 +285,7 @@ class CLanguage(object): "cmake", "cmake_ninja_vs2019", "cmake_vs2019", + "cmake_vs2022", ], ) _check_arch(self.args.arch, ["default", "x64", "x86"]) @@ -301,6 +302,8 @@ class CLanguage(object): activate_vs_tools = "2019" elif self.args.compiler == "cmake_vs2019": cmake_generator = "Visual Studio 16 2019" + elif self.args.compiler == "cmake_vs2022": + cmake_generator = "Visual Studio 17 2022" else: print("should never reach here.") sys.exit(1) @@ -1684,6 +1687,7 @@ argp.add_argument( "cmake", "cmake_ninja_vs2019", "cmake_vs2019", + "cmake_vs2022", "mono", ], default="default", From 2ca568663fb957c50e77f8ebfdd7a4a5a1f4c605 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Wed, 14 Feb 2024 13:31:01 -0800 Subject: [PATCH 19/37] [CI] Upgraded the docker images for Ruby Distrib Tests (#35911) - Dropped Ruby 2.7 test (Ruby 2.7 hit EOL 1 year ago) - Upgraded the base image of Ruby docker images from Debian:10 to Debian:11 (Debian:10 hit EOL) Closes #35911 PiperOrigin-RevId: 607088383 --- .../Dockerfile.template | 3 +- .../Dockerfile.template | 3 +- .../Dockerfile.template | 3 +- .../Dockerfile.template | 3 +- .../grpc_interop_ruby/Dockerfile.template | 6 +-- templates/tools/dockerfile/ruby_deps.include | 17 -------- .../dockerfile/rvm_mkdir_workaround.include | 3 ++ .../ruby_debian11_arm64/Dockerfile.template | 3 +- .../ruby_debian11_x64/Dockerfile.template | 3 +- .../dockerimage_current_versions.bzl | 16 ++++--- .../ruby_debian10_x64.current_version | 1 - .../distribtest/ruby_debian10_x64/Dockerfile | 19 --------- ...ruby_debian10_x64_ruby_2_7.current_version | 1 - .../ruby_debian10_x64_ruby_2_7/Dockerfile | 42 ------------------- ...ruby_debian10_x64_ruby_3_0.current_version | 1 - ...ruby_debian10_x64_ruby_3_1.current_version | 1 - ...ruby_debian10_x64_ruby_3_2.current_version | 1 - ...ruby_debian10_x64_ruby_3_3.current_version | 1 - ...ruby_debian11_x64_ruby_3_0.current_version | 1 + .../Dockerfile | 6 ++- ...ruby_debian11_x64_ruby_3_1.current_version | 1 + .../Dockerfile | 6 ++- ...ruby_debian11_x64_ruby_3_2.current_version | 1 + .../Dockerfile | 6 ++- ...ruby_debian11_x64_ruby_3_3.current_version | 1 + .../Dockerfile | 6 ++- .../grpc_interop_ruby.current_version | 2 +- .../interoptest/grpc_interop_ruby/Dockerfile | 15 ++++--- .../test/ruby_debian11_arm64.current_version | 2 +- .../test/ruby_debian11_arm64/Dockerfile | 13 +++--- .../test/ruby_debian11_x64.current_version | 2 +- .../test/ruby_debian11_x64/Dockerfile | 13 +++--- .../artifacts/distribtest_targets.py | 12 +++--- 33 files changed, 83 insertions(+), 131 deletions(-) rename templates/tools/dockerfile/distribtest/{ruby_debian10_x64_ruby_3_0 => ruby_debian11_x64_ruby_3_0}/Dockerfile.template (91%) rename templates/tools/dockerfile/distribtest/{ruby_debian10_x64_ruby_3_1 => ruby_debian11_x64_ruby_3_1}/Dockerfile.template (91%) rename templates/tools/dockerfile/distribtest/{ruby_debian10_x64_ruby_3_2 => ruby_debian11_x64_ruby_3_2}/Dockerfile.template (91%) rename templates/tools/dockerfile/distribtest/{ruby_debian10_x64_ruby_3_3 => ruby_debian11_x64_ruby_3_3}/Dockerfile.template (91%) delete mode 100644 templates/tools/dockerfile/ruby_deps.include create mode 100644 templates/tools/dockerfile/rvm_mkdir_workaround.include delete mode 100644 tools/dockerfile/distribtest/ruby_debian10_x64.current_version delete mode 100644 tools/dockerfile/distribtest/ruby_debian10_x64/Dockerfile delete mode 100644 tools/dockerfile/distribtest/ruby_debian10_x64_ruby_2_7.current_version delete mode 100644 tools/dockerfile/distribtest/ruby_debian10_x64_ruby_2_7/Dockerfile delete mode 100644 tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_0.current_version delete mode 100644 tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_1.current_version delete mode 100644 tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_2.current_version delete mode 100644 tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_3.current_version create mode 100644 tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_0.current_version rename tools/dockerfile/distribtest/{ruby_debian10_x64_ruby_3_0 => ruby_debian11_x64_ruby_3_0}/Dockerfile (93%) create mode 100644 tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_1.current_version rename tools/dockerfile/distribtest/{ruby_debian10_x64_ruby_3_1 => ruby_debian11_x64_ruby_3_1}/Dockerfile (93%) create mode 100644 tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_2.current_version rename tools/dockerfile/distribtest/{ruby_debian10_x64_ruby_3_2 => ruby_debian11_x64_ruby_3_2}/Dockerfile (93%) create mode 100644 tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_3.current_version rename tools/dockerfile/distribtest/{ruby_debian10_x64_ruby_3_3 => ruby_debian11_x64_ruby_3_3}/Dockerfile (93%) diff --git a/templates/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_0/Dockerfile.template b/templates/tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_0/Dockerfile.template similarity index 91% rename from templates/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_0/Dockerfile.template rename to templates/tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_0/Dockerfile.template index 2bff360192c..b8da129ddb3 100644 --- a/templates/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_0/Dockerfile.template +++ b/templates/tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_0/Dockerfile.template @@ -14,10 +14,11 @@ # See the License for the specific language governing permissions and # limitations under the License. - FROM debian:10 + FROM debian:11 <%include file="../../apt_get_basic.include"/> <%include file="../../ruby_3_0_deps.include"/> + <%include file="../../rvm_mkdir_workaround.include"/> # Define the default command. CMD ["bash"] diff --git a/templates/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_1/Dockerfile.template b/templates/tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_1/Dockerfile.template similarity index 91% rename from templates/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_1/Dockerfile.template rename to templates/tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_1/Dockerfile.template index a0f67ff7d89..a9e4abe237d 100644 --- a/templates/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_1/Dockerfile.template +++ b/templates/tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_1/Dockerfile.template @@ -14,10 +14,11 @@ # See the License for the specific language governing permissions and # limitations under the License. - FROM debian:10 + FROM debian:11 <%include file="../../apt_get_basic.include"/> <%include file="../../ruby_3_1_deps.include"/> + <%include file="../../rvm_mkdir_workaround.include"/> # Define the default command. CMD ["bash"] diff --git a/templates/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_2/Dockerfile.template b/templates/tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_2/Dockerfile.template similarity index 91% rename from templates/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_2/Dockerfile.template rename to templates/tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_2/Dockerfile.template index a19720ff436..35575ac4c68 100644 --- a/templates/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_2/Dockerfile.template +++ b/templates/tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_2/Dockerfile.template @@ -14,10 +14,11 @@ # See the License for the specific language governing permissions and # limitations under the License. - FROM debian:10 + FROM debian:11 <%include file="../../apt_get_basic.include"/> <%include file="../../ruby_3_2_deps.include"/> + <%include file="../../rvm_mkdir_workaround.include"/> # Define the default command. CMD ["bash"] diff --git a/templates/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_3/Dockerfile.template b/templates/tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_3/Dockerfile.template similarity index 91% rename from templates/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_3/Dockerfile.template rename to templates/tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_3/Dockerfile.template index 8fc44ef1a80..dcdcf5a5b6d 100644 --- a/templates/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_3/Dockerfile.template +++ b/templates/tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_3/Dockerfile.template @@ -14,10 +14,11 @@ # See the License for the specific language governing permissions and # limitations under the License. - FROM debian:10 + FROM debian:11 <%include file="../../apt_get_basic.include"/> <%include file="../../ruby_3_3_deps.include"/> + <%include file="../../rvm_mkdir_workaround.include"/> # Define the default command. CMD ["bash"] diff --git a/templates/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile.template b/templates/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile.template index e53ac1c7ff1..e9689394bb1 100644 --- a/templates/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile.template +++ b/templates/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile.template @@ -18,11 +18,11 @@ <%include file="../../apt_get_basic.include"/> <%include file="../../run_tests_python_deps.include"/> - <%include file="../../ruby_deps.include"/> + <%include file="../../ruby_3_0_deps.include"/> + <%include file="../../rvm_mkdir_workaround.include"/> <%include file="../../cmake.include"/> <%include file="../../ccache.include"/> <%include file="../../run_tests_addons.include"/> # Define the default command. - CMD ["bash"] - + CMD ["bash"] \ No newline at end of file diff --git a/templates/tools/dockerfile/ruby_deps.include b/templates/tools/dockerfile/ruby_deps.include deleted file mode 100644 index a57130f9bcc..00000000000 --- a/templates/tools/dockerfile/ruby_deps.include +++ /dev/null @@ -1,17 +0,0 @@ -#================== -# Ruby dependencies - -# Install rvm -RUN apt-get update && apt-get install -y gnupg2 && apt-get clean -RUN gpg2 --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB -RUN \curl -sSL https://get.rvm.io | bash -s stable - -# Install Ruby 2.7 -RUN apt-get update && apt-get install -y procps && apt-get clean -# "--disable-binary" is a workaround for https://github.com/rvm/rvm/issues/4975 -RUN /bin/bash -l -c "rvm install ruby-2.7 --disable-binary" -RUN /bin/bash -l -c "rvm use --default ruby-2.7" -RUN /bin/bash -l -c "echo 'gem: --no-document' > ~/.gemrc" -RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc" -RUN /bin/bash -l -c "echo 'rvm --default use ruby-2.7' >> ~/.bashrc" -RUN /bin/bash -l -c "gem install bundler --no-document" diff --git a/templates/tools/dockerfile/rvm_mkdir_workaround.include b/templates/tools/dockerfile/rvm_mkdir_workaround.include new file mode 100644 index 00000000000..07a4e87a708 --- /dev/null +++ b/templates/tools/dockerfile/rvm_mkdir_workaround.include @@ -0,0 +1,3 @@ +# Workaround for rvm unable to find mkdir properly +# https://stackoverflow.com/questions/64653051 +RUN ln -s /bin/mkdir /usr/bin/mkdir diff --git a/templates/tools/dockerfile/test/ruby_debian11_arm64/Dockerfile.template b/templates/tools/dockerfile/test/ruby_debian11_arm64/Dockerfile.template index dd7db3b1133..077ce99d860 100644 --- a/templates/tools/dockerfile/test/ruby_debian11_arm64/Dockerfile.template +++ b/templates/tools/dockerfile/test/ruby_debian11_arm64/Dockerfile.template @@ -18,7 +18,8 @@ <%include file="../../apt_get_basic.include"/> <%include file="../../run_tests_python_deps.include"/> - <%include file="../../ruby_deps.include"/> + <%include file="../../ruby_3_0_deps.include"/> + <%include file="../../rvm_mkdir_workaround.include"/> <%include file="../../cmake.include"/> <%include file="../../ccache.include"/> <%include file="../../run_tests_addons.include"/> diff --git a/templates/tools/dockerfile/test/ruby_debian11_x64/Dockerfile.template b/templates/tools/dockerfile/test/ruby_debian11_x64/Dockerfile.template index 9d24d767888..f8a01a3cdb5 100644 --- a/templates/tools/dockerfile/test/ruby_debian11_x64/Dockerfile.template +++ b/templates/tools/dockerfile/test/ruby_debian11_x64/Dockerfile.template @@ -18,7 +18,8 @@ <%include file="../../apt_get_basic.include"/> <%include file="../../run_tests_python_deps.include"/> - <%include file="../../ruby_deps.include"/> + <%include file="../../ruby_3_0_deps.include"/> + <%include file="../../rvm_mkdir_workaround.include"/> <%include file="../../cmake.include"/> <%include file="../../ccache.include"/> <%include file="../../run_tests_addons.include"/> diff --git a/tools/bazelify_tests/dockerimage_current_versions.bzl b/tools/bazelify_tests/dockerimage_current_versions.bzl index 638fdcaada4..281016b5b8d 100644 --- a/tools/bazelify_tests/dockerimage_current_versions.bzl +++ b/tools/bazelify_tests/dockerimage_current_versions.bzl @@ -55,12 +55,10 @@ DOCKERIMAGE_CURRENT_VERSIONS = { "tools/dockerfile/distribtest/python_ubuntu2004_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_ubuntu2004_x64@sha256:342e9dc23b674ad256b220745745be818708a1baa25a2690f0d4f777e28a22a3", "tools/dockerfile/distribtest/python_ubuntu2204_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_ubuntu2204_x64@sha256:b6ca497348741615406d1d571cf5042008da934c4f708877aa06a66b5dc3036c", "tools/dockerfile/distribtest/ruby_centos7_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_centos7_x64@sha256:4d529b984b78ca179086f7f9b416605e2d9a96ca0a28a71f4421bb5ffdc18f96", - "tools/dockerfile/distribtest/ruby_debian10_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian10_x64@sha256:1298c39c950b2a48261555b6cff1ae66230a5020f100d3b381759285f0caf84e", - "tools/dockerfile/distribtest/ruby_debian10_x64_ruby_2_7.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian10_x64_ruby_2_7@sha256:5ee26ad3abe2683c9a8ee03987ab0ae63f50793c3d3f5e4be6e6cbacb4556fcf", - "tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_0.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian10_x64_ruby_3_0@sha256:be9dfa852af7fe5c700747bbad5b93d15c3f030ac980a107e98fc9b70fee6077", - "tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_1.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian10_x64_ruby_3_1@sha256:282a052275fa9e0378301d759f5febcc9f1ff0ebb702e536690cd4fed1a43904", - "tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_2.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian10_x64_ruby_3_2@sha256:f1ba9030d7d7496a6ed42c0e1a9a2fe3204208e94983d7d4b6b4c7752042ed28", - "tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_3.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian10_x64_ruby_3_3@sha256:a26d6bfe8f799ae817db0bd0280283c2cc9ea0f02174fe9a2cb0cd10d71d81f8", + "tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_0.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_x64_ruby_3_0@sha256:ed855e283001ea9f526aea3830e93c2138194e1740cf6aa5e11a19f6d81e0e22", + "tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_1.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_x64_ruby_3_1@sha256:17e3f48a1e1ec2caade5ce4db73f92d6f1f27b34d7cba61968d8b3e3d15f9ca3", + "tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_2.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_x64_ruby_3_2@sha256:aa5bf946f601a3ee557be63816428e6a4c997f33194c1a58499d0ca17e7bf562", + "tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_3.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_x64_ruby_3_3@sha256:7643e5e91e9f24dc403ad967372c6a6b9a92ccb98fddaa5c29087e5e4ddb1f11", "tools/dockerfile/distribtest/ruby_ubuntu2004_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_ubuntu2004_x64@sha256:426cbf625df0c0e7451b9716041996dc6a35a3788c1a24d68891256f84733d8e", "tools/dockerfile/distribtest/ruby_ubuntu2204_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_ubuntu2204_x64@sha256:1c74c312f8a4ab37e629732a35daa3056e12136b90f37540cdf9fa11303c7eb8", "tools/dockerfile/grpc_artifact_centos6_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_centos6_x64@sha256:3285047265ea2b7c5d4df4c769b2d05f56288d947c75e16d27ae2dee693f791b", @@ -87,7 +85,7 @@ DOCKERIMAGE_CURRENT_VERSIONS = { "tools/dockerfile/interoptest/grpc_interop_php7.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_php7@sha256:9342ff81689c37d9e79fd6abcc08bf310eb48174e83bd3bfce39d225c02f0d4e", "tools/dockerfile/interoptest/grpc_interop_python.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_python@sha256:1f55faacfb4be587e6d26b05561e79bf3e17fe81c69a990e8aeca4257081c9ac", "tools/dockerfile/interoptest/grpc_interop_pythonasyncio.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_pythonasyncio@sha256:47127a7863097b436613885a8866a2ef055470452838ceebb31f692ac88ac1d1", - "tools/dockerfile/interoptest/grpc_interop_ruby.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_ruby@sha256:7b044d6848f82234dba81b38d8eca220b608f830f93b42932df59ed6fe20b24d", + "tools/dockerfile/interoptest/grpc_interop_ruby.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_ruby@sha256:efd7f41a736dd4b8f73b32f5215b86f6bfe9013c422dfcd77978de0253aaee45", "tools/dockerfile/interoptest/lb_interop_fake_servers.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/lb_interop_fake_servers@sha256:b89a51dd9147e1293f50ee64dd719fce5929ca7894d3770a3d80dbdecb99fd52", "tools/dockerfile/test/android_ndk.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/android_ndk@sha256:2866e815ceaf7407a9017842b86c1bf1efc96abe77ecff88e932aa1b1ddf727e", "tools/dockerfile/test/bazel.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/bazel@sha256:32bde2dcb2087f2a32afab59e4dfedf7e8c76a52c69881f63a239d311f0e5ecf", @@ -112,7 +110,7 @@ DOCKERIMAGE_CURRENT_VERSIONS = { "tools/dockerfile/test/python_debian11_default_arm64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_debian11_default_arm64@sha256:fccca33a655c7aa89dd7ebd9492cbcc1f636bd2a004cd939d1982cfce3d68326", "tools/dockerfile/test/python_debian11_default_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_debian11_default_x64@sha256:a956ff4bfbfa4fc6a00c7113f2dada7ce8cf4011f236f07197732981875d6519", "tools/dockerfile/test/rbe_ubuntu2004.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/rbe_ubuntu2004@sha256:d3951aeadf43e3bee6adc5b86d26cdaf0b9d1b5baf790d7b2530d1c197adc9f8", - "tools/dockerfile/test/ruby_debian11_arm64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_arm64@sha256:7e77cf17e2e8657f4cc23ac9f93630bf13213fff961799e0f16dae17cd45cf6d", - "tools/dockerfile/test/ruby_debian11_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_x64@sha256:e4cb502caccf2db733268ce2ddc951fda8a9df2f7f53d6b74523c33d40c83006", + "tools/dockerfile/test/ruby_debian11_arm64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_arm64@sha256:d2e79919b2e2d4cc36a29682ecb5170641df4fb506cfb453978ffdeb8a841bd9", + "tools/dockerfile/test/ruby_debian11_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_x64@sha256:f8fc0ec22065278e5bc02ad7f9a68191e46d083035b3a90ed587561dba9c58c5", "tools/dockerfile/test/sanity.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/sanity@sha256:ca65b29302e8e381aabf9deb1952281f4e9963654e19c3d681148b5aa972c8a2", } diff --git a/tools/dockerfile/distribtest/ruby_debian10_x64.current_version b/tools/dockerfile/distribtest/ruby_debian10_x64.current_version deleted file mode 100644 index 8bfb13b7315..00000000000 --- a/tools/dockerfile/distribtest/ruby_debian10_x64.current_version +++ /dev/null @@ -1 +0,0 @@ -us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian10_x64:0efa3a484833ce2b8960ad449d62571a8d595f65@sha256:1298c39c950b2a48261555b6cff1ae66230a5020f100d3b381759285f0caf84e \ No newline at end of file diff --git a/tools/dockerfile/distribtest/ruby_debian10_x64/Dockerfile b/tools/dockerfile/distribtest/ruby_debian10_x64/Dockerfile deleted file mode 100644 index bc78b44b1b1..00000000000 --- a/tools/dockerfile/distribtest/ruby_debian10_x64/Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2015 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM debian:10 - -RUN apt-get update && apt-get install -y ruby-full - -RUN gem install bundler -v 1.17.3 --no-document diff --git a/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_2_7.current_version b/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_2_7.current_version deleted file mode 100644 index a9be0430e59..00000000000 --- a/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_2_7.current_version +++ /dev/null @@ -1 +0,0 @@ -us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian10_x64_ruby_2_7:9a09a0a915e62249957791eda907a011078c5aea@sha256:5ee26ad3abe2683c9a8ee03987ab0ae63f50793c3d3f5e4be6e6cbacb4556fcf \ No newline at end of file diff --git a/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_2_7/Dockerfile b/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_2_7/Dockerfile deleted file mode 100644 index 678822a73f4..00000000000 --- a/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_2_7/Dockerfile +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2020 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM debian:10 - -# Install Git and basic packages. -RUN apt-get update && apt-get install -y \ - gnupg2 \ - procps \ - curl \ - gcc && apt-get clean - -#================== -# Ruby dependencies - -# Install rvm -RUN gpg2 --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB -RUN \curl -sSL https://get.rvm.io | bash -s stable - -# Install Ruby 2.7 -RUN /bin/bash -l -c "rvm install ruby-2.7.0" -RUN /bin/bash -l -c "rvm use --default ruby-2.7.0" -RUN /bin/bash -l -c "echo 'gem: --no-document' > ~/.gemrc" -RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc" -RUN /bin/bash -l -c "echo 'rvm --default use ruby-2.7.0' >> ~/.bashrc" -RUN /bin/bash -l -c "gem install bundler --no-document" - -RUN mkdir /var/local/jenkins - -# Define the default command. -CMD ["bash"] diff --git a/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_0.current_version b/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_0.current_version deleted file mode 100644 index 709c65daebb..00000000000 --- a/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_0.current_version +++ /dev/null @@ -1 +0,0 @@ -us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian10_x64_ruby_3_0:dce91a018316b35b962e5fa576fded4775457077@sha256:be9dfa852af7fe5c700747bbad5b93d15c3f030ac980a107e98fc9b70fee6077 \ No newline at end of file diff --git a/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_1.current_version b/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_1.current_version deleted file mode 100644 index 4995cf042c1..00000000000 --- a/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_1.current_version +++ /dev/null @@ -1 +0,0 @@ -us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian10_x64_ruby_3_1:348a4b975f4d19772bb9a1a95fc03720f89edbc0@sha256:282a052275fa9e0378301d759f5febcc9f1ff0ebb702e536690cd4fed1a43904 \ No newline at end of file diff --git a/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_2.current_version b/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_2.current_version deleted file mode 100644 index f17c0cea7c5..00000000000 --- a/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_2.current_version +++ /dev/null @@ -1 +0,0 @@ -us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian10_x64_ruby_3_2:e9f8640c6bb80a029c65ed6ebdfddd9574a041e4@sha256:f1ba9030d7d7496a6ed42c0e1a9a2fe3204208e94983d7d4b6b4c7752042ed28 \ No newline at end of file diff --git a/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_3.current_version b/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_3.current_version deleted file mode 100644 index bbe14a70714..00000000000 --- a/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_3.current_version +++ /dev/null @@ -1 +0,0 @@ -us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian10_x64_ruby_3_3:c063f13c0987355593ce6bcf2123bb9717eeba35@sha256:a26d6bfe8f799ae817db0bd0280283c2cc9ea0f02174fe9a2cb0cd10d71d81f8 \ No newline at end of file diff --git a/tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_0.current_version b/tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_0.current_version new file mode 100644 index 00000000000..4af2537fbb4 --- /dev/null +++ b/tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_0.current_version @@ -0,0 +1 @@ +us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_x64_ruby_3_0:ddf5f067d1e4ae46f4178ef14d2ea2cea6ae6c76@sha256:ed855e283001ea9f526aea3830e93c2138194e1740cf6aa5e11a19f6d81e0e22 \ No newline at end of file diff --git a/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_0/Dockerfile b/tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_0/Dockerfile similarity index 93% rename from tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_0/Dockerfile rename to tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_0/Dockerfile index 639bd90158c..0bebd8eabb7 100644 --- a/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_0/Dockerfile +++ b/tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_0/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM debian:10 +FROM debian:11 #================= # Basic C core dependencies @@ -79,6 +79,10 @@ RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc" RUN /bin/bash -l -c "echo 'rvm --default use ruby-3.0.5' >> ~/.bashrc" RUN /bin/bash -l -c "gem install bundler --no-document" +# Workaround for rvm unable to find mkdir properly +# https://stackoverflow.com/questions/64653051 +RUN ln -s /bin/mkdir /usr/bin/mkdir + # Define the default command. CMD ["bash"] diff --git a/tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_1.current_version b/tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_1.current_version new file mode 100644 index 00000000000..f4d4d464ffd --- /dev/null +++ b/tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_1.current_version @@ -0,0 +1 @@ +us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_x64_ruby_3_1:d6f1ff5cee79b1c3829573d15ea68c5b453f2f8d@sha256:17e3f48a1e1ec2caade5ce4db73f92d6f1f27b34d7cba61968d8b3e3d15f9ca3 \ No newline at end of file diff --git a/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_1/Dockerfile b/tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_1/Dockerfile similarity index 93% rename from tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_1/Dockerfile rename to tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_1/Dockerfile index 314b0d889fb..2bd1f138737 100644 --- a/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_1/Dockerfile +++ b/tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_1/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM debian:10 +FROM debian:11 #================= # Basic C core dependencies @@ -79,6 +79,10 @@ RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc" RUN /bin/bash -l -c "echo 'rvm --default use ruby-3.1.3' >> ~/.bashrc" RUN /bin/bash -l -c "gem install bundler --no-document" +# Workaround for rvm unable to find mkdir properly +# https://stackoverflow.com/questions/64653051 +RUN ln -s /bin/mkdir /usr/bin/mkdir + # Define the default command. CMD ["bash"] diff --git a/tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_2.current_version b/tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_2.current_version new file mode 100644 index 00000000000..2e3cefbb0aa --- /dev/null +++ b/tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_2.current_version @@ -0,0 +1 @@ +us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_x64_ruby_3_2:88a99c45a0bc5fedd959731d78817781b0b545bc@sha256:aa5bf946f601a3ee557be63816428e6a4c997f33194c1a58499d0ca17e7bf562 \ No newline at end of file diff --git a/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_2/Dockerfile b/tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_2/Dockerfile similarity index 93% rename from tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_2/Dockerfile rename to tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_2/Dockerfile index 546acc6d0d3..c0f6f1a6e73 100644 --- a/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_2/Dockerfile +++ b/tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_2/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM debian:10 +FROM debian:11 #================= # Basic C core dependencies @@ -79,6 +79,10 @@ RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc" RUN /bin/bash -l -c "echo 'rvm --default use ruby-3.2.0' >> ~/.bashrc" RUN /bin/bash -l -c "gem install bundler --no-document" +# Workaround for rvm unable to find mkdir properly +# https://stackoverflow.com/questions/64653051 +RUN ln -s /bin/mkdir /usr/bin/mkdir + # Define the default command. CMD ["bash"] diff --git a/tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_3.current_version b/tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_3.current_version new file mode 100644 index 00000000000..387848baa6e --- /dev/null +++ b/tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_3.current_version @@ -0,0 +1 @@ +us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_x64_ruby_3_3:3fc462641e0ecd88ddd38a3ab14fa70c291f98b0@sha256:7643e5e91e9f24dc403ad967372c6a6b9a92ccb98fddaa5c29087e5e4ddb1f11 \ No newline at end of file diff --git a/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_3/Dockerfile b/tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_3/Dockerfile similarity index 93% rename from tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_3/Dockerfile rename to tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_3/Dockerfile index f7f27b649f7..dba3a43c844 100644 --- a/tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_3/Dockerfile +++ b/tools/dockerfile/distribtest/ruby_debian11_x64_ruby_3_3/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM debian:10 +FROM debian:11 #================= # Basic C core dependencies @@ -79,6 +79,10 @@ RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc" RUN /bin/bash -l -c "echo 'rvm --default use ruby-3.3.0' >> ~/.bashrc" RUN /bin/bash -l -c "gem install bundler --no-document" +# Workaround for rvm unable to find mkdir properly +# https://stackoverflow.com/questions/64653051 +RUN ln -s /bin/mkdir /usr/bin/mkdir + # Define the default command. CMD ["bash"] diff --git a/tools/dockerfile/interoptest/grpc_interop_ruby.current_version b/tools/dockerfile/interoptest/grpc_interop_ruby.current_version index e682c81bed0..a9a2b901f64 100644 --- a/tools/dockerfile/interoptest/grpc_interop_ruby.current_version +++ b/tools/dockerfile/interoptest/grpc_interop_ruby.current_version @@ -1 +1 @@ -us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_ruby:b234d5eeaa79e9dff66484a1a76d142786064a9c@sha256:7b044d6848f82234dba81b38d8eca220b608f830f93b42932df59ed6fe20b24d \ No newline at end of file +us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_ruby:28d14e44cd85d7dbe814c72156be140ab120beec@sha256:efd7f41a736dd4b8f73b32f5215b86f6bfe9013c422dfcd77978de0253aaee45 \ No newline at end of file diff --git a/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile index 484aa78050c..6aab5501ed4 100644 --- a/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile @@ -95,16 +95,19 @@ RUN apt-get update && apt-get install -y gnupg2 && apt-get clean RUN gpg2 --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB RUN \curl -sSL https://get.rvm.io | bash -s stable -# Install Ruby 2.7 +# Install Ruby 3.0 RUN apt-get update && apt-get install -y procps && apt-get clean -# "--disable-binary" is a workaround for https://github.com/rvm/rvm/issues/4975 -RUN /bin/bash -l -c "rvm install ruby-2.7 --disable-binary" -RUN /bin/bash -l -c "rvm use --default ruby-2.7" +RUN /bin/bash -l -c "rvm install ruby-3.0.5" +RUN /bin/bash -l -c "rvm use --default ruby-3.0.5" RUN /bin/bash -l -c "echo 'gem: --no-document' > ~/.gemrc" RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc" -RUN /bin/bash -l -c "echo 'rvm --default use ruby-2.7' >> ~/.bashrc" +RUN /bin/bash -l -c "echo 'rvm --default use ruby-3.0.5' >> ~/.bashrc" RUN /bin/bash -l -c "gem install bundler --no-document" +# Workaround for rvm unable to find mkdir properly +# https://stackoverflow.com/questions/64653051 +RUN ln -s /bin/mkdir /usr/bin/mkdir + #================= # Install cmake # Note that this step should be only used for distributions that have new enough cmake to satisfy gRPC's cmake version requirement. @@ -130,4 +133,4 @@ RUN mkdir /var/local/jenkins # Define the default command. -CMD ["bash"] +CMD ["bash"] \ No newline at end of file diff --git a/tools/dockerfile/test/ruby_debian11_arm64.current_version b/tools/dockerfile/test/ruby_debian11_arm64.current_version index f435625c598..d114eb7fd32 100644 --- a/tools/dockerfile/test/ruby_debian11_arm64.current_version +++ b/tools/dockerfile/test/ruby_debian11_arm64.current_version @@ -1 +1 @@ -us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_arm64:8db47f0dd21c60760dbc6aede225a29e8b5eb469@sha256:7e77cf17e2e8657f4cc23ac9f93630bf13213fff961799e0f16dae17cd45cf6d \ No newline at end of file +us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_arm64:710eb956a7f5b98b0e2265dd156d946926a142d9@sha256:d2e79919b2e2d4cc36a29682ecb5170641df4fb506cfb453978ffdeb8a841bd9 \ No newline at end of file diff --git a/tools/dockerfile/test/ruby_debian11_arm64/Dockerfile b/tools/dockerfile/test/ruby_debian11_arm64/Dockerfile index 1d0e2b418e3..08e09b5d09a 100644 --- a/tools/dockerfile/test/ruby_debian11_arm64/Dockerfile +++ b/tools/dockerfile/test/ruby_debian11_arm64/Dockerfile @@ -95,16 +95,19 @@ RUN apt-get update && apt-get install -y gnupg2 && apt-get clean RUN gpg2 --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB RUN \curl -sSL https://get.rvm.io | bash -s stable -# Install Ruby 2.7 +# Install Ruby 3.0 RUN apt-get update && apt-get install -y procps && apt-get clean -# "--disable-binary" is a workaround for https://github.com/rvm/rvm/issues/4975 -RUN /bin/bash -l -c "rvm install ruby-2.7 --disable-binary" -RUN /bin/bash -l -c "rvm use --default ruby-2.7" +RUN /bin/bash -l -c "rvm install ruby-3.0.5" +RUN /bin/bash -l -c "rvm use --default ruby-3.0.5" RUN /bin/bash -l -c "echo 'gem: --no-document' > ~/.gemrc" RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc" -RUN /bin/bash -l -c "echo 'rvm --default use ruby-2.7' >> ~/.bashrc" +RUN /bin/bash -l -c "echo 'rvm --default use ruby-3.0.5' >> ~/.bashrc" RUN /bin/bash -l -c "gem install bundler --no-document" +# Workaround for rvm unable to find mkdir properly +# https://stackoverflow.com/questions/64653051 +RUN ln -s /bin/mkdir /usr/bin/mkdir + #================= # Install cmake # Note that this step should be only used for distributions that have new enough cmake to satisfy gRPC's cmake version requirement. diff --git a/tools/dockerfile/test/ruby_debian11_x64.current_version b/tools/dockerfile/test/ruby_debian11_x64.current_version index efcd2708432..cbed14c1281 100644 --- a/tools/dockerfile/test/ruby_debian11_x64.current_version +++ b/tools/dockerfile/test/ruby_debian11_x64.current_version @@ -1 +1 @@ -us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_x64:db4549f16eb39f5e16241e9b434f2cf0df018fc5@sha256:e4cb502caccf2db733268ce2ddc951fda8a9df2f7f53d6b74523c33d40c83006 \ No newline at end of file +us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_x64:6dfb957e5de39c71c1ca45338a975f7f98bd59d7@sha256:f8fc0ec22065278e5bc02ad7f9a68191e46d083035b3a90ed587561dba9c58c5 \ No newline at end of file diff --git a/tools/dockerfile/test/ruby_debian11_x64/Dockerfile b/tools/dockerfile/test/ruby_debian11_x64/Dockerfile index 796d53a7b46..c9c43f73bd9 100644 --- a/tools/dockerfile/test/ruby_debian11_x64/Dockerfile +++ b/tools/dockerfile/test/ruby_debian11_x64/Dockerfile @@ -95,16 +95,19 @@ RUN apt-get update && apt-get install -y gnupg2 && apt-get clean RUN gpg2 --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB RUN \curl -sSL https://get.rvm.io | bash -s stable -# Install Ruby 2.7 +# Install Ruby 3.0 RUN apt-get update && apt-get install -y procps && apt-get clean -# "--disable-binary" is a workaround for https://github.com/rvm/rvm/issues/4975 -RUN /bin/bash -l -c "rvm install ruby-2.7 --disable-binary" -RUN /bin/bash -l -c "rvm use --default ruby-2.7" +RUN /bin/bash -l -c "rvm install ruby-3.0.5" +RUN /bin/bash -l -c "rvm use --default ruby-3.0.5" RUN /bin/bash -l -c "echo 'gem: --no-document' > ~/.gemrc" RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc" -RUN /bin/bash -l -c "echo 'rvm --default use ruby-2.7' >> ~/.bashrc" +RUN /bin/bash -l -c "echo 'rvm --default use ruby-3.0.5' >> ~/.bashrc" RUN /bin/bash -l -c "gem install bundler --no-document" +# Workaround for rvm unable to find mkdir properly +# https://stackoverflow.com/questions/64653051 +RUN ln -s /bin/mkdir /usr/bin/mkdir + #================= # Install cmake # Note that this step should be only used for distributions that have new enough cmake to satisfy gRPC's cmake version requirement. diff --git a/tools/run_tests/artifacts/distribtest_targets.py b/tools/run_tests/artifacts/distribtest_targets.py index fe71900756f..c367171c331 100644 --- a/tools/run_tests/artifacts/distribtest_targets.py +++ b/tools/run_tests/artifacts/distribtest_targets.py @@ -468,22 +468,22 @@ def targets(): RubyDistribTest( "linux", "x64", - "debian10", - ruby_version="ruby_2_7", + "debian11", + ruby_version="ruby_3_2", source=True, presubmit=True, ), RubyDistribTest( - "linux", "x64", "debian10", ruby_version="ruby_3_0", presubmit=True + "linux", "x64", "debian11", ruby_version="ruby_3_0", presubmit=True ), RubyDistribTest( - "linux", "x64", "debian10", ruby_version="ruby_3_1", presubmit=True + "linux", "x64", "debian11", ruby_version="ruby_3_1", presubmit=True ), RubyDistribTest( - "linux", "x64", "debian10", ruby_version="ruby_3_2", presubmit=True + "linux", "x64", "debian11", ruby_version="ruby_3_2", presubmit=True ), RubyDistribTest( - "linux", "x64", "debian10", ruby_version="ruby_3_3", presubmit=True + "linux", "x64", "debian11", ruby_version="ruby_3_3", presubmit=True ), RubyDistribTest("linux", "x64", "centos7"), RubyDistribTest("linux", "x64", "ubuntu2004"), From bb3edd22b36fb7aaea53666c8b010c2a9e316c80 Mon Sep 17 00:00:00 2001 From: Xuan Wang Date: Wed, 14 Feb 2024 16:41:36 -0800 Subject: [PATCH 20/37] [Python Modernization] Deprecate pkg_resources (#35849) Discuss thread about this change: [link](https://mail.google.com/mail/u/0/#sent/QgrcJHsBpWNGRlrMktwbppGGfFTVCFLcQgL?compose=new) Closes #35849 PiperOrigin-RevId: 607144827 --- setup.py | 17 ------------ src/python/grpcio/README.rst | 26 ------------------- src/python/grpcio_observability/setup.py | 17 ------------ .../protoc_plugin/_split_definitions_test.py | 25 ++++++++++++++++-- tools/distrib/python/grpcio_tools/README.rst | 26 ------------------- .../python/grpcio_tools/grpc_tools/command.py | 26 ++++++++++++++++--- tools/distrib/python/grpcio_tools/setup.py | 17 ------------ tools/distrib/python/xds_protos/build.py | 25 ++++++++++++++++-- 8 files changed, 68 insertions(+), 111 deletions(-) diff --git a/setup.py b/setup.py index 6ebfae1811e..2ce5fef4223 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,6 @@ import sys import sysconfig import _metadata -import pkg_resources from setuptools import Extension from setuptools.command import egg_info @@ -454,22 +453,6 @@ if "linux" in sys.platform or "darwin" in sys.platform: DEFINE_MACROS += (("PyMODINIT_FUNC", pymodinit),) DEFINE_MACROS += (("GRPC_POSIX_FORK_ALLOW_PTHREAD_ATFORK", 1),) -# By default, Python3 distutils enforces compatibility of -# c plugins (.so files) with the OSX version Python was built with. -# We need OSX 10.10, the oldest which supports C++ thread_local. -# Python 3.9: Mac OS Big Sur sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET') returns int (11) -if "darwin" in sys.platform: - mac_target = sysconfig.get_config_var("MACOSX_DEPLOYMENT_TARGET") - if mac_target: - mac_target = pkg_resources.parse_version(str(mac_target)) - if mac_target < pkg_resources.parse_version("10.10.0"): - os.environ["MACOSX_DEPLOYMENT_TARGET"] = "10.10" - os.environ["_PYTHON_HOST_PLATFORM"] = re.sub( - r"macosx-[0-9]+\.[0-9]+-(.+)", - r"macosx-10.10-\1", - sysconfig.get_platform(), - ) - def cython_extensions_and_necessity(): cython_module_files = [ diff --git a/src/python/grpcio/README.rst b/src/python/grpcio/README.rst index 8e1826b0192..f3563e2e803 100644 --- a/src/python/grpcio/README.rst +++ b/src/python/grpcio/README.rst @@ -72,32 +72,6 @@ Troubleshooting Help, I ... -* **... see a** :code:`pkg_resources.VersionConflict` **when I try to install - grpc** - - This is likely because :code:`pip` doesn't own the offending dependency, - which in turn is likely because your operating system's package manager owns - it. You'll need to force the installation of the dependency: - - :code:`pip install --ignore-installed $OFFENDING_DEPENDENCY` - - For example, if you get an error like the following: - - :: - - Traceback (most recent call last): - File "", line 17, in - ... - File "/usr/lib/python2.7/dist-packages/pkg_resources.py", line 509, in find - raise VersionConflict(dist, req) - pkg_resources.VersionConflict: (six 1.8.0 (/usr/lib/python2.7/dist-packages), Requirement.parse('six>=1.10')) - - You can fix it by doing: - - :: - - sudo pip install --ignore-installed six - * **... see the following error on some platforms** :: diff --git a/src/python/grpcio_observability/setup.py b/src/python/grpcio_observability/setup.py index 4d4ebbeec57..f9bd1966504 100644 --- a/src/python/grpcio_observability/setup.py +++ b/src/python/grpcio_observability/setup.py @@ -22,7 +22,6 @@ from subprocess import PIPE import sys import sysconfig -import pkg_resources import setuptools from setuptools import Extension from setuptools.command import build_ext @@ -206,22 +205,6 @@ if "linux" in sys.platform or "darwin" in sys.platform: pymodinit = 'extern "C" __attribute__((visibility ("default"))) PyObject*' DEFINE_MACROS += (("PyMODINIT_FUNC", pymodinit),) -# By default, Python3 distutils enforces compatibility of -# c plugins (.so files) with the OSX version Python was built with. -# We need OSX 10.10, the oldest which supports C++ thread_local. -if "darwin" in sys.platform: - mac_target = sysconfig.get_config_var("MACOSX_DEPLOYMENT_TARGET") - if mac_target and ( - pkg_resources.parse_version(mac_target) - < pkg_resources.parse_version("10.10.0") - ): - os.environ["MACOSX_DEPLOYMENT_TARGET"] = "10.10" - os.environ["_PYTHON_HOST_PLATFORM"] = re.sub( - r"macosx-[0-9]+\.[0-9]+-(.+)", - r"macosx-10.10-\1", - sysconfig.get_platform(), - ) - def extension_modules(): if BUILD_WITH_CYTHON: diff --git a/src/python/grpcio_tests/tests/protoc_plugin/_split_definitions_test.py b/src/python/grpcio_tests/tests/protoc_plugin/_split_definitions_test.py index 9b8c3acf948..17e752adb42 100644 --- a/src/python/grpcio_tests/tests/protoc_plugin/_split_definitions_test.py +++ b/src/python/grpcio_tests/tests/protoc_plugin/_split_definitions_test.py @@ -26,7 +26,12 @@ import unittest import grpc from grpc_tools import protoc -import pkg_resources + +if sys.version_info >= (3, 9, 0): + from importlib import resources +else: + import pkg_resources + from tests.unit import test_common @@ -48,6 +53,22 @@ def _system_path(path_insertion): sys.path = old_system_path +def _get_resource_file_name( + package_or_requirement: str, resource_name: str +) -> str: + """Obtain the filename for a resource on the file system.""" + file_name = None + if sys.version_info >= (3, 9, 0): + file_name = ( + resources.files(package_or_requirement) / resource_name + ).resolve() + else: + file_name = pkg_resources.resource_filename( + package_or_requirement, resource_name + ) + return str(file_name) + + # NOTE(nathaniel): https://twitter.com/exoplaneteer/status/677259364256747520 # Life lesson "just always default to idempotence" reinforced. def _create_directory_tree(root, path_components_sequence): @@ -367,7 +388,7 @@ class WellKnownTypesTest(unittest.TestCase): def testWellKnownTypes(self): os.chdir(_TEST_DIR) out_dir = tempfile.mkdtemp(suffix="wkt_test", dir=".") - well_known_protos_include = pkg_resources.resource_filename( + well_known_protos_include = _get_resource_file_name( "grpc_tools", "_proto" ) args = [ diff --git a/tools/distrib/python/grpcio_tools/README.rst b/tools/distrib/python/grpcio_tools/README.rst index f0b5240f73c..05db96e33d5 100644 --- a/tools/distrib/python/grpcio_tools/README.rst +++ b/tools/distrib/python/grpcio_tools/README.rst @@ -75,32 +75,6 @@ Troubleshooting Help, I ... -* **... see a** :code:`pkg_resources.VersionConflict` **when I try to install - grpc** - - This is likely because :code:`pip` doesn't own the offending dependency, - which in turn is likely because your operating system's package manager owns - it. You'll need to force the installation of the dependency: - - :code:`pip install --ignore-installed $OFFENDING_DEPENDENCY` - - For example, if you get an error like the following: - - :: - - Traceback (most recent call last): - File "", line 17, in - ... - File "/usr/lib/python2.7/dist-packages/pkg_resources.py", line 509, in find - raise VersionConflict(dist, req) - pkg_resources.VersionConflict: (six 1.8.0 (/usr/lib/python2.7/dist-packages), Requirement.parse('six>=1.10')) - - You can fix it by doing: - - :: - - sudo pip install --ignore-installed six - * **... see compiler errors on some platforms when either installing from source or from the source distribution** If you see diff --git a/tools/distrib/python/grpcio_tools/grpc_tools/command.py b/tools/distrib/python/grpcio_tools/grpc_tools/command.py index 5b0b7eb6a79..33e43ffc6fb 100644 --- a/tools/distrib/python/grpcio_tools/grpc_tools/command.py +++ b/tools/distrib/python/grpcio_tools/grpc_tools/command.py @@ -16,9 +16,29 @@ import os import sys from grpc_tools import protoc -import pkg_resources import setuptools +if sys.version_info >= (3, 9, 0): + from importlib import resources +else: + import pkg_resources + + +def _get_resource_file_name( + package_or_requirement: str, resource_name: str +) -> str: + """Obtain the filename for a resource on the file system.""" + file_name = None + if sys.version_info >= (3, 9, 0): + file_name = ( + resources.files(package_or_requirement) / resource_name + ).resolve() + else: + file_name = pkg_resources.resource_filename( + package_or_requirement, resource_name + ) + return str(file_name) + def build_package_protos(package_root, strict_mode=False): proto_files = [] @@ -30,9 +50,7 @@ def build_package_protos(package_root, strict_mode=False): os.path.abspath(os.path.join(root, filename)) ) - well_known_protos_include = pkg_resources.resource_filename( - "grpc_tools", "_proto" - ) + well_known_protos_include = _get_resource_file_name("grpc_tools", "_proto") for proto_file in proto_files: command = [ diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py index b479143387d..ee026efd00c 100644 --- a/tools/distrib/python/grpcio_tools/setup.py +++ b/tools/distrib/python/grpcio_tools/setup.py @@ -24,7 +24,6 @@ from subprocess import PIPE import sys import sysconfig -import pkg_resources import setuptools from setuptools import Extension from setuptools.command import build_ext @@ -214,22 +213,6 @@ if "win32" in sys.platform: elif "linux" in sys.platform or "darwin" in sys.platform: DEFINE_MACROS += (("HAVE_PTHREAD", 1),) -# By default, Python3 setuptools(distutils) enforces compatibility of -# c plugins (.so files) with the OSX version Python was built with. -# We need OSX 10.10, the oldest which supports C++ thread_local. -if "darwin" in sys.platform: - mac_target = sysconfig.get_config_var("MACOSX_DEPLOYMENT_TARGET") - if mac_target and ( - pkg_resources.parse_version(mac_target) - < pkg_resources.parse_version("10.10.0") - ): - os.environ["MACOSX_DEPLOYMENT_TARGET"] = "10.10" - os.environ["_PYTHON_HOST_PLATFORM"] = re.sub( - r"macosx-[0-9]+\.[0-9]+-(.+)", - r"macosx-10.10-\1", - sysconfig.get_platform(), - ) - def package_data(): tools_path = GRPC_PYTHON_TOOLS_PACKAGE.replace(".", os.path.sep) diff --git a/tools/distrib/python/xds_protos/build.py b/tools/distrib/python/xds_protos/build.py index cb5cc9bb1b7..9c1d2e9061e 100644 --- a/tools/distrib/python/xds_protos/build.py +++ b/tools/distrib/python/xds_protos/build.py @@ -15,15 +15,36 @@ """Builds the content of xds-protos package""" import os +import sys from grpc_tools import protoc -import pkg_resources + +if sys.version_info >= (3, 9, 0): + from importlib import resources +else: + import pkg_resources def localize_path(p): return os.path.join(*p.split("/")) +def _get_resource_file_name( + package_or_requirement: str, resource_name: str +) -> str: + """Obtain the filename for a resource on the file system.""" + file_name = None + if sys.version_info >= (3, 9, 0): + file_name = ( + resources.files(package_or_requirement) / resource_name + ).resolve() + else: + file_name = pkg_resources.resource_filename( + package_or_requirement, resource_name + ) + return str(file_name) + + # We might not want to compile all the protos EXCLUDE_PROTO_PACKAGES_LIST = tuple( localize_path(p) @@ -46,7 +67,7 @@ OPENCENSUS_PROTO_ROOT = os.path.join( GRPC_ROOT, "third_party", "opencensus-proto", "src" ) OPENTELEMETRY_PROTO_ROOT = os.path.join(GRPC_ROOT, "third_party", "opentelemetry") -WELL_KNOWN_PROTOS_INCLUDE = pkg_resources.resource_filename("grpc_tools", "_proto") +WELL_KNOWN_PROTOS_INCLUDE = _get_resource_file_name("grpc_tools", "_proto") OUTPUT_PATH = WORK_DIR From ccc8017bf04cdb17bc81170dd148ee07b8c89820 Mon Sep 17 00:00:00 2001 From: Yousuk Seung Date: Thu, 15 Feb 2024 10:34:04 -0800 Subject: [PATCH 21/37] enable work_serializer_dispatch (#35891) Closes #35891 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/35891 from yousukseung:work_serializer ca36cde216e944c67f8f0b39c5af327afc3cd264 PiperOrigin-RevId: 607380601 --- bazel/experiments.bzl | 2 +- src/core/lib/experiments/experiments.cc | 15 ++++++++++++--- src/core/lib/experiments/experiments.yaml | 5 +++-- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/bazel/experiments.bzl b/bazel/experiments.bzl index 5281d7d5b07..0e5e67f0028 100644 --- a/bazel/experiments.bzl +++ b/bazel/experiments.bzl @@ -54,7 +54,7 @@ EXPERIMENT_ENABLES = { "v3_compression_filter": "v3_compression_filter", "v3_server_auth_filter": "v3_server_auth_filter", "work_serializer_clears_time_cache": "work_serializer_clears_time_cache", - "work_serializer_dispatch": "work_serializer_dispatch", + "work_serializer_dispatch": "event_engine_client,work_serializer_dispatch", "write_size_policy": "write_size_policy", "write_size_cap": "write_size_cap,write_size_policy", "wrr_delegate_to_pick_first": "wrr_delegate_to_pick_first", diff --git a/src/core/lib/experiments/experiments.cc b/src/core/lib/experiments/experiments.cc index ac4dcd85889..f08d1413185 100644 --- a/src/core/lib/experiments/experiments.cc +++ b/src/core/lib/experiments/experiments.cc @@ -173,6 +173,8 @@ const char* const description_work_serializer_dispatch = "callback, instead of running things inline in the first thread that " "successfully enqueues work."; const char* const additional_constraints_work_serializer_dispatch = "{}"; +const uint8_t required_experiments_work_serializer_dispatch[] = { + static_cast(grpc_core::kExperimentIdEventEngineClient)}; const char* const description_write_size_policy = "Try to size writes such that they don't create too large of a backlog"; const char* const additional_constraints_write_size_policy = "{}"; @@ -287,7 +289,8 @@ const ExperimentMetadata g_experiment_metadata[] = { additional_constraints_work_serializer_clears_time_cache, nullptr, 0, true, true}, {"work_serializer_dispatch", description_work_serializer_dispatch, - additional_constraints_work_serializer_dispatch, nullptr, 0, false, true}, + additional_constraints_work_serializer_dispatch, + required_experiments_work_serializer_dispatch, 1, false, true}, {"write_size_policy", description_write_size_policy, additional_constraints_write_size_policy, nullptr, 0, true, true}, {"write_size_cap", description_write_size_cap, @@ -450,6 +453,8 @@ const char* const description_work_serializer_dispatch = "callback, instead of running things inline in the first thread that " "successfully enqueues work."; const char* const additional_constraints_work_serializer_dispatch = "{}"; +const uint8_t required_experiments_work_serializer_dispatch[] = { + static_cast(grpc_core::kExperimentIdEventEngineClient)}; const char* const description_write_size_policy = "Try to size writes such that they don't create too large of a backlog"; const char* const additional_constraints_write_size_policy = "{}"; @@ -564,7 +569,8 @@ const ExperimentMetadata g_experiment_metadata[] = { additional_constraints_work_serializer_clears_time_cache, nullptr, 0, true, true}, {"work_serializer_dispatch", description_work_serializer_dispatch, - additional_constraints_work_serializer_dispatch, nullptr, 0, false, true}, + additional_constraints_work_serializer_dispatch, + required_experiments_work_serializer_dispatch, 1, false, true}, {"write_size_policy", description_write_size_policy, additional_constraints_write_size_policy, nullptr, 0, true, true}, {"write_size_cap", description_write_size_cap, @@ -727,6 +733,8 @@ const char* const description_work_serializer_dispatch = "callback, instead of running things inline in the first thread that " "successfully enqueues work."; const char* const additional_constraints_work_serializer_dispatch = "{}"; +const uint8_t required_experiments_work_serializer_dispatch[] = { + static_cast(grpc_core::kExperimentIdEventEngineClient)}; const char* const description_write_size_policy = "Try to size writes such that they don't create too large of a backlog"; const char* const additional_constraints_write_size_policy = "{}"; @@ -841,7 +849,8 @@ const ExperimentMetadata g_experiment_metadata[] = { additional_constraints_work_serializer_clears_time_cache, nullptr, 0, true, true}, {"work_serializer_dispatch", description_work_serializer_dispatch, - additional_constraints_work_serializer_dispatch, nullptr, 0, false, true}, + additional_constraints_work_serializer_dispatch, + required_experiments_work_serializer_dispatch, 1, false, true}, {"write_size_policy", description_write_size_policy, additional_constraints_write_size_policy, nullptr, 0, true, true}, {"write_size_cap", description_write_size_cap, diff --git a/src/core/lib/experiments/experiments.yaml b/src/core/lib/experiments/experiments.yaml index 74b2d14eada..87eeaa32f13 100644 --- a/src/core/lib/experiments/experiments.yaml +++ b/src/core/lib/experiments/experiments.yaml @@ -287,9 +287,10 @@ Have the work serializer dispatch work to event engine for every callback, instead of running things inline in the first thread that successfully enqueues work. - expiry: 2024/02/10 - owner: ctiller@google.com + expiry: 2024/03/31 + owner: ysseung@google.com test_tags: ["core_end2end_test", "cpp_end2end_test", "xds_end2end_test", "lb_unit_test"] + requires: ["event_engine_client"] - name: write_size_cap description: Limit outgoing writes proportional to the target write size From cf79445171d206683cab669ba588288fbe354285 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 15 Feb 2024 14:46:59 -0800 Subject: [PATCH 22/37] [chaotic-good] Fix channel creation (#35907) Closes #35907 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/35907 from ctiller:cucumber+carrot 185f65afb8862a8b26ab4076f5fc6066e49aee37 PiperOrigin-RevId: 607462304 --- CMakeLists.txt | 187 +++++++++++++++++---------- build_autogenerated.yaml | 128 +++++++++++------- src/cpp/ext/chaotic_good.cc | 1 - test/cpp/ext/BUILD | 39 ++++++ test/cpp/ext/chaotic_good_test.cc | 38 ++++++ tools/run_tests/generated/tests.json | 64 ++++++--- 6 files changed, 325 insertions(+), 132 deletions(-) create mode 100644 test/cpp/ext/BUILD create mode 100644 test/cpp/ext/chaotic_good_test.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index e1cdd443aff..88429d02709 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -953,9 +953,6 @@ if(gRPC_BUILD_TESTS) add_dependencies(buildtests_cxx channel_trace_test) add_dependencies(buildtests_cxx channelz_registry_test) add_dependencies(buildtests_cxx channelz_service_test) - if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_POSIX) - add_dependencies(buildtests_cxx chaotic_good_test) - endif() add_dependencies(buildtests_cxx check_gcp_environment_linux_test) add_dependencies(buildtests_cxx check_gcp_environment_windows_test) add_dependencies(buildtests_cxx chunked_vector_test) @@ -1424,8 +1421,12 @@ if(gRPC_BUILD_TESTS) add_dependencies(buildtests_cxx test_core_security_ssl_credentials_test) add_dependencies(buildtests_cxx test_core_slice_slice_buffer_test) add_dependencies(buildtests_cxx test_core_slice_slice_test) + if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_POSIX) + add_dependencies(buildtests_cxx test_core_transport_test_suite_chaotic_good_test) + endif() add_dependencies(buildtests_cxx test_cpp_client_credentials_test) add_dependencies(buildtests_cxx test_cpp_end2end_ssl_credentials_test) + add_dependencies(buildtests_cxx test_cpp_ext_chaotic_good_test) add_dependencies(buildtests_cxx test_cpp_server_credentials_test) add_dependencies(buildtests_cxx test_cpp_util_slice_test) add_dependencies(buildtests_cxx test_cpp_util_time_test) @@ -10000,69 +10001,6 @@ target_link_libraries(channelz_service_test ) -endif() -if(gRPC_BUILD_TESTS) -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_POSIX) - - add_executable(chaotic_good_test - ${_gRPC_PROTO_GENS_DIR}/test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.pb.cc - ${_gRPC_PROTO_GENS_DIR}/test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.grpc.pb.cc - ${_gRPC_PROTO_GENS_DIR}/test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.pb.h - ${_gRPC_PROTO_GENS_DIR}/test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.grpc.pb.h - src/core/ext/transport/chaotic_good/chaotic_good_transport.cc - src/core/ext/transport/chaotic_good/client_transport.cc - src/core/ext/transport/chaotic_good/frame.cc - src/core/ext/transport/chaotic_good/frame_header.cc - src/core/ext/transport/chaotic_good/server_transport.cc - src/core/lib/transport/promise_endpoint.cc - test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.cc - test/core/transport/test_suite/call_content.cc - test/core/transport/test_suite/call_shapes.cc - test/core/transport/test_suite/chaotic_good_fixture.cc - test/core/transport/test_suite/fixture.cc - test/core/transport/test_suite/no_op.cc - test/core/transport/test_suite/stress.cc - test/core/transport/test_suite/test.cc - test/core/transport/test_suite/test_main.cc - ) - if(WIN32 AND MSVC) - if(BUILD_SHARED_LIBS) - target_compile_definitions(chaotic_good_test - PRIVATE - "GPR_DLL_IMPORTS" - "GRPC_DLL_IMPORTS" - ) - endif() - endif() - target_compile_features(chaotic_good_test PUBLIC cxx_std_14) - target_include_directories(chaotic_good_test - PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}/include - ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} - ${_gRPC_RE2_INCLUDE_DIR} - ${_gRPC_SSL_INCLUDE_DIR} - ${_gRPC_UPB_GENERATED_DIR} - ${_gRPC_UPB_GRPC_GENERATED_DIR} - ${_gRPC_UPB_INCLUDE_DIR} - ${_gRPC_XXHASH_INCLUDE_DIR} - ${_gRPC_ZLIB_INCLUDE_DIR} - third_party/googletest/googletest/include - third_party/googletest/googletest - third_party/googletest/googlemock/include - third_party/googletest/googlemock - ${_gRPC_PROTO_GENS_DIR} - ) - - target_link_libraries(chaotic_good_test - ${_gRPC_ALLTARGETS_LIBRARIES} - gtest - ${_gRPC_PROTOBUF_LIBRARIES} - grpc_test_util - ) - - -endif() endif() if(gRPC_BUILD_TESTS) @@ -29848,6 +29786,69 @@ target_link_libraries(test_core_slice_slice_test ) +endif() +if(gRPC_BUILD_TESTS) +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_POSIX) + + add_executable(test_core_transport_test_suite_chaotic_good_test + ${_gRPC_PROTO_GENS_DIR}/test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.pb.cc + ${_gRPC_PROTO_GENS_DIR}/test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.pb.h + ${_gRPC_PROTO_GENS_DIR}/test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.grpc.pb.h + src/core/ext/transport/chaotic_good/chaotic_good_transport.cc + src/core/ext/transport/chaotic_good/client_transport.cc + src/core/ext/transport/chaotic_good/frame.cc + src/core/ext/transport/chaotic_good/frame_header.cc + src/core/ext/transport/chaotic_good/server_transport.cc + src/core/lib/transport/promise_endpoint.cc + test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.cc + test/core/transport/test_suite/call_content.cc + test/core/transport/test_suite/call_shapes.cc + test/core/transport/test_suite/chaotic_good_fixture.cc + test/core/transport/test_suite/fixture.cc + test/core/transport/test_suite/no_op.cc + test/core/transport/test_suite/stress.cc + test/core/transport/test_suite/test.cc + test/core/transport/test_suite/test_main.cc + ) + if(WIN32 AND MSVC) + if(BUILD_SHARED_LIBS) + target_compile_definitions(test_core_transport_test_suite_chaotic_good_test + PRIVATE + "GPR_DLL_IMPORTS" + "GRPC_DLL_IMPORTS" + ) + endif() + endif() + target_compile_features(test_core_transport_test_suite_chaotic_good_test PUBLIC cxx_std_14) + target_include_directories(test_core_transport_test_suite_chaotic_good_test + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} + third_party/googletest/googletest/include + third_party/googletest/googletest + third_party/googletest/googlemock/include + third_party/googletest/googlemock + ${_gRPC_PROTO_GENS_DIR} + ) + + target_link_libraries(test_core_transport_test_suite_chaotic_good_test + ${_gRPC_ALLTARGETS_LIBRARIES} + gtest + ${_gRPC_PROTOBUF_LIBRARIES} + grpc_test_util + ) + + +endif() endif() if(gRPC_BUILD_TESTS) @@ -29953,6 +29954,60 @@ target_link_libraries(test_cpp_end2end_ssl_credentials_test ) +endif() +if(gRPC_BUILD_TESTS) + +add_executable(test_cpp_ext_chaotic_good_test + src/core/ext/transport/chaotic_good/chaotic_good_transport.cc + src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc + src/core/ext/transport/chaotic_good/client_transport.cc + src/core/ext/transport/chaotic_good/frame.cc + src/core/ext/transport/chaotic_good/frame_header.cc + src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc + src/core/ext/transport/chaotic_good/server_transport.cc + src/core/ext/transport/chaotic_good/settings_metadata.cc + src/core/lib/transport/promise_endpoint.cc + src/cpp/ext/chaotic_good.cc + test/cpp/ext/chaotic_good_test.cc +) +if(WIN32 AND MSVC) + if(BUILD_SHARED_LIBS) + target_compile_definitions(test_cpp_ext_chaotic_good_test + PRIVATE + "GPR_DLL_IMPORTS" + "GRPC_DLL_IMPORTS" + "GRPCXX_DLL_IMPORTS" + ) + endif() +endif() +target_compile_features(test_cpp_ext_chaotic_good_test PUBLIC cxx_std_14) +target_include_directories(test_cpp_ext_chaotic_good_test + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} + third_party/googletest/googletest/include + third_party/googletest/googletest + third_party/googletest/googlemock/include + third_party/googletest/googlemock + ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(test_cpp_ext_chaotic_good_test + ${_gRPC_ALLTARGETS_LIBRARIES} + gtest + grpc++ + grpc_test_util +) + + endif() if(gRPC_BUILD_TESTS) diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index 5a5a3362e01..452e24b9d99 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -7574,51 +7574,6 @@ targets: - gtest - grpcpp_channelz - grpc++_test_util -- name: chaotic_good_test - gtest: true - build: test - language: c++ - headers: - - src/core/ext/transport/chaotic_good/chaotic_good_transport.h - - src/core/ext/transport/chaotic_good/client_transport.h - - src/core/ext/transport/chaotic_good/frame.h - - src/core/ext/transport/chaotic_good/frame_header.h - - src/core/ext/transport/chaotic_good/server_transport.h - - src/core/lib/promise/event_engine_wakeup_scheduler.h - - src/core/lib/promise/inter_activity_latch.h - - src/core/lib/promise/inter_activity_pipe.h - - src/core/lib/promise/mpsc.h - - src/core/lib/promise/switch.h - - src/core/lib/promise/wait_set.h - - src/core/lib/transport/promise_endpoint.h - - test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.h - - test/core/transport/test_suite/fixture.h - - test/core/transport/test_suite/test.h - src: - - test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.proto - - src/core/ext/transport/chaotic_good/chaotic_good_transport.cc - - src/core/ext/transport/chaotic_good/client_transport.cc - - src/core/ext/transport/chaotic_good/frame.cc - - src/core/ext/transport/chaotic_good/frame_header.cc - - src/core/ext/transport/chaotic_good/server_transport.cc - - src/core/lib/transport/promise_endpoint.cc - - test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.cc - - test/core/transport/test_suite/call_content.cc - - test/core/transport/test_suite/call_shapes.cc - - test/core/transport/test_suite/chaotic_good_fixture.cc - - test/core/transport/test_suite/fixture.cc - - test/core/transport/test_suite/no_op.cc - - test/core/transport/test_suite/stress.cc - - test/core/transport/test_suite/test.cc - - test/core/transport/test_suite/test_main.cc - deps: - - gtest - - protobuf - - grpc_test_util - platforms: - - linux - - posix - uses_polling: false - name: check_gcp_environment_linux_test gtest: true build: test @@ -18868,6 +18823,51 @@ targets: - gtest - grpc uses_polling: false +- name: test_core_transport_test_suite_chaotic_good_test + gtest: true + build: test + language: c++ + headers: + - src/core/ext/transport/chaotic_good/chaotic_good_transport.h + - src/core/ext/transport/chaotic_good/client_transport.h + - src/core/ext/transport/chaotic_good/frame.h + - src/core/ext/transport/chaotic_good/frame_header.h + - src/core/ext/transport/chaotic_good/server_transport.h + - src/core/lib/promise/event_engine_wakeup_scheduler.h + - src/core/lib/promise/inter_activity_latch.h + - src/core/lib/promise/inter_activity_pipe.h + - src/core/lib/promise/mpsc.h + - src/core/lib/promise/switch.h + - src/core/lib/promise/wait_set.h + - src/core/lib/transport/promise_endpoint.h + - test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.h + - test/core/transport/test_suite/fixture.h + - test/core/transport/test_suite/test.h + src: + - test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.proto + - src/core/ext/transport/chaotic_good/chaotic_good_transport.cc + - src/core/ext/transport/chaotic_good/client_transport.cc + - src/core/ext/transport/chaotic_good/frame.cc + - src/core/ext/transport/chaotic_good/frame_header.cc + - src/core/ext/transport/chaotic_good/server_transport.cc + - src/core/lib/transport/promise_endpoint.cc + - test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.cc + - test/core/transport/test_suite/call_content.cc + - test/core/transport/test_suite/call_shapes.cc + - test/core/transport/test_suite/chaotic_good_fixture.cc + - test/core/transport/test_suite/fixture.cc + - test/core/transport/test_suite/no_op.cc + - test/core/transport/test_suite/stress.cc + - test/core/transport/test_suite/test.cc + - test/core/transport/test_suite/test_main.cc + deps: + - gtest + - protobuf + - grpc_test_util + platforms: + - linux + - posix + uses_polling: false - name: test_cpp_client_credentials_test gtest: true build: test @@ -18897,6 +18897,44 @@ targets: deps: - gtest - grpc++_test_util +- name: test_cpp_ext_chaotic_good_test + gtest: true + build: test + language: c++ + headers: + - src/core/ext/transport/chaotic_good/chaotic_good_transport.h + - src/core/ext/transport/chaotic_good/client/chaotic_good_connector.h + - src/core/ext/transport/chaotic_good/client_transport.h + - src/core/ext/transport/chaotic_good/frame.h + - src/core/ext/transport/chaotic_good/frame_header.h + - src/core/ext/transport/chaotic_good/server/chaotic_good_server.h + - src/core/ext/transport/chaotic_good/server_transport.h + - src/core/ext/transport/chaotic_good/settings_metadata.h + - src/core/lib/promise/event_engine_wakeup_scheduler.h + - src/core/lib/promise/inter_activity_latch.h + - src/core/lib/promise/inter_activity_pipe.h + - src/core/lib/promise/mpsc.h + - src/core/lib/promise/switch.h + - src/core/lib/promise/wait_for_callback.h + - src/core/lib/promise/wait_set.h + - src/core/lib/transport/promise_endpoint.h + - src/cpp/ext/chaotic_good.h + src: + - src/core/ext/transport/chaotic_good/chaotic_good_transport.cc + - src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc + - src/core/ext/transport/chaotic_good/client_transport.cc + - src/core/ext/transport/chaotic_good/frame.cc + - src/core/ext/transport/chaotic_good/frame_header.cc + - src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc + - src/core/ext/transport/chaotic_good/server_transport.cc + - src/core/ext/transport/chaotic_good/settings_metadata.cc + - src/core/lib/transport/promise_endpoint.cc + - src/cpp/ext/chaotic_good.cc + - test/cpp/ext/chaotic_good_test.cc + deps: + - gtest + - grpc++ + - grpc_test_util - name: test_cpp_server_credentials_test gtest: true build: test diff --git a/src/cpp/ext/chaotic_good.cc b/src/cpp/ext/chaotic_good.cc index 8711a73ace0..4c9beac14e9 100644 --- a/src/cpp/ext/chaotic_good.cc +++ b/src/cpp/ext/chaotic_good.cc @@ -44,7 +44,6 @@ class ChaoticGoodInsecureChannelCredentialsImpl final auto channel = grpc::CreateChannelInternal( "", grpc_chaotic_good_channel_create(target.c_str(), &channel_args), std::move(interceptor_creators)); - grpc_channel_args_destroy(&channel_args); return channel; } diff --git a/test/cpp/ext/BUILD b/test/cpp/ext/BUILD new file mode 100644 index 00000000000..ecdba7d8f7e --- /dev/null +++ b/test/cpp/ext/BUILD @@ -0,0 +1,39 @@ +# Copyright 2024 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("//bazel:grpc_build_system.bzl", "grpc_cc_test", "grpc_package") + +licenses(["notice"]) + +grpc_package( + name = "test/cpp/ext", + visibility = "tests", +) + +grpc_cc_test( + name = "chaotic_good_test", + srcs = [ + "chaotic_good_test.cc", + ], + external_deps = [ + "gtest", + ], + language = "C++", + tags = [], + deps = [ + "//:grpc++", + "//:grpcpp_chaotic_good", + "//test/core/util:grpc_test_util", + ], +) diff --git a/test/cpp/ext/chaotic_good_test.cc b/test/cpp/ext/chaotic_good_test.cc new file mode 100644 index 00000000000..71e49a5a71c --- /dev/null +++ b/test/cpp/ext/chaotic_good_test.cc @@ -0,0 +1,38 @@ +// Copyright 2024 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/cpp/ext/chaotic_good.h" + +#include "gtest/gtest.h" + +#include + +#include "test/core/util/test_config.h" + +namespace grpc { +namespace { + +TEST(ChaoticGoodTest, CreateChannel) { + auto creds = ChaoticGoodInsecureChannelCredentials(); + auto channel = CreateChannel("localhost:50051", creds); +} + +} // namespace +} // namespace grpc + +int main(int argc, char** argv) { + grpc::testing::TestEnvironment env(&argc, argv); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index fc1aae76b97..5bdfdcb4496 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -1945,26 +1945,6 @@ ], "uses_polling": true }, - { - "args": [], - "benchmark": false, - "ci_platforms": [ - "linux", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "gtest": true, - "language": "c++", - "name": "chaotic_good_test", - "platforms": [ - "linux", - "posix" - ], - "uses_polling": false - }, { "args": [], "benchmark": false, @@ -10531,6 +10511,26 @@ ], "uses_polling": false }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": true, + "language": "c++", + "name": "test_core_transport_test_suite_chaotic_good_test", + "platforms": [ + "linux", + "posix" + ], + "uses_polling": false + }, { "args": [], "benchmark": false, @@ -10579,6 +10579,30 @@ ], "uses_polling": true }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": true, + "language": "c++", + "name": "test_cpp_ext_chaotic_good_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "uses_polling": true + }, { "args": [], "benchmark": false, From 2c9b599e5e0f8b740c9b97d3faf58d0729aab68b Mon Sep 17 00:00:00 2001 From: Xuan Wang Date: Thu, 15 Feb 2024 15:32:11 -0800 Subject: [PATCH 23/37] [Python O11Y] Reapply registered method change (#35850) This reverts commit a18279db2e77415fa78a643c7d2404649f531496. Closes #35850 PiperOrigin-RevId: 607476066 --- examples/python/helloworld/helloworld_pb2.py | 27 ++--- examples/python/helloworld/helloworld_pb2.pyi | 14 +-- .../python/helloworld/helloworld_pb2_grpc.py | 104 +++++++++++++++++- src/compiler/python_generator.cc | 23 ++-- src/python/.gitignore | 4 +- src/python/grpcio/grpc/__init__.py | 12 ++ src/python/grpcio/grpc/_channel.py | 75 ++++++++++++- .../grpc/_cython/_cygrpc/channel.pxd.pxi | 7 ++ .../grpc/_cython/_cygrpc/channel.pyx.pxi | 71 +++++++++--- .../grpcio/grpc/_cython/_cygrpc/grpc.pxi | 6 + src/python/grpcio/grpc/_interceptor.py | 36 +++++- src/python/grpcio/grpc/_simple_stubs.py | 81 +++++++++++--- src/python/grpcio/grpc/aio/_base_channel.py | 12 ++ src/python/grpcio/grpc/aio/_channel.py | 21 ++++ .../grpc_testing/_channel/_channel.py | 27 ++++- .../tests/channelz/_channelz_servicer_test.py | 17 ++- .../grpcio_tests/tests/csds/csds_test.py | 5 +- .../protoc_plugin/_python_plugin_test.py | 11 ++ .../tests/qps/benchmark_client.py | 9 +- .../tests/status/_grpc_status_test.py | 25 ++++- .../grpcio_tests/tests/unit/_abort_test.py | 20 +++- .../tests/unit/_auth_context_test.py | 20 +++- .../tests/unit/_channel_close_test.py | 35 ++++-- .../tests/unit/_compression_test.py | 20 +++- .../unit/_contextvars_propagation_test.py | 10 +- .../tests/unit/_dns_resolver_test.py | 5 +- .../tests/unit/_empty_message_test.py | 24 ++-- .../unit/_error_message_encoding_test.py | 5 +- .../tests/unit/_exit_scenarios.py | 20 +++- .../tests/unit/_interceptor_test.py | 8 +- .../tests/unit/_invalid_metadata_test.py | 9 +- .../tests/unit/_invocation_defects_test.py | 22 +++- .../tests/unit/_local_credentials_test.py | 14 ++- .../tests/unit/_metadata_code_details_test.py | 80 ++++++++------ .../tests/unit/_metadata_flags_test.py | 39 +++++-- .../grpcio_tests/tests/unit/_metadata_test.py | 16 ++- .../tests/unit/_reconnect_test.py | 5 +- .../tests/unit/_resource_exhausted_test.py | 20 +++- .../tests/unit/_rpc_test_helpers.py | 18 ++- .../tests/unit/_server_shutdown_scenarios.py | 5 +- .../tests/unit/_session_cache_test.py | 5 +- .../grpcio_tests/tests/unit/_signal_client.py | 24 ++-- .../tests/unit/_xds_credentials_test.py | 14 ++- .../tests_gevent/unit/close_channel_test.py | 2 + .../tests_py3_only/unit/_leak_test.py | 5 +- .../tests_py3_only/unit/_simple_stubs_test.py | 46 +++++++- 46 files changed, 863 insertions(+), 215 deletions(-) diff --git a/examples/python/helloworld/helloworld_pb2.py b/examples/python/helloworld/helloworld_pb2.py index f5b4f2d27dc..6ee31ad94a2 100644 --- a/examples/python/helloworld/helloworld_pb2.py +++ b/examples/python/helloworld/helloworld_pb2.py @@ -1,11 +1,12 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: helloworld.proto +# Protobuf Python Version: 4.25.0 """Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -13,18 +14,18 @@ _sym_db = _symbol_database.Default() -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x10helloworld.proto\x12\nhelloworld\"\x1c\n\x0cHelloRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"\x1d\n\nHelloReply\x12\x0f\n\x07message\x18\x01 \x01(\t2I\n\x07Greeter\x12>\n\x08SayHello\x12\x18.helloworld.HelloRequest\x1a\x16.helloworld.HelloReply\"\x00\x42\x36\n\x1bio.grpc.examples.helloworldB\x0fHelloWorldProtoP\x01\xa2\x02\x03HLWb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x10helloworld.proto\x12\nhelloworld\"\x1c\n\x0cHelloRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"\x1d\n\nHelloReply\x12\x0f\n\x07message\x18\x01 \x01(\t2\xe4\x01\n\x07Greeter\x12>\n\x08SayHello\x12\x18.helloworld.HelloRequest\x1a\x16.helloworld.HelloReply\"\x00\x12K\n\x13SayHelloStreamReply\x12\x18.helloworld.HelloRequest\x1a\x16.helloworld.HelloReply\"\x00\x30\x01\x12L\n\x12SayHelloBidiStream\x12\x18.helloworld.HelloRequest\x1a\x16.helloworld.HelloReply\"\x00(\x01\x30\x01\x42\x36\n\x1bio.grpc.examples.helloworldB\x0fHelloWorldProtoP\x01\xa2\x02\x03HLWb\x06proto3') -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'helloworld_pb2', globals()) +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'helloworld_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: - - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'\n\033io.grpc.examples.helloworldB\017HelloWorldProtoP\001\242\002\003HLW' - _HELLOREQUEST._serialized_start=32 - _HELLOREQUEST._serialized_end=60 - _HELLOREPLY._serialized_start=62 - _HELLOREPLY._serialized_end=91 - _GREETER._serialized_start=93 - _GREETER._serialized_end=166 + _globals['DESCRIPTOR']._options = None + _globals['DESCRIPTOR']._serialized_options = b'\n\033io.grpc.examples.helloworldB\017HelloWorldProtoP\001\242\002\003HLW' + _globals['_HELLOREQUEST']._serialized_start=32 + _globals['_HELLOREQUEST']._serialized_end=60 + _globals['_HELLOREPLY']._serialized_start=62 + _globals['_HELLOREPLY']._serialized_end=91 + _globals['_GREETER']._serialized_start=94 + _globals['_GREETER']._serialized_end=322 # @@protoc_insertion_point(module_scope) diff --git a/examples/python/helloworld/helloworld_pb2.pyi b/examples/python/helloworld/helloworld_pb2.pyi index 8c4b5b22805..bf0bd395ad5 100644 --- a/examples/python/helloworld/helloworld_pb2.pyi +++ b/examples/python/helloworld/helloworld_pb2.pyi @@ -4,14 +4,14 @@ from typing import ClassVar as _ClassVar, Optional as _Optional DESCRIPTOR: _descriptor.FileDescriptor -class HelloReply(_message.Message): - __slots__ = ["message"] - MESSAGE_FIELD_NUMBER: _ClassVar[int] - message: str - def __init__(self, message: _Optional[str] = ...) -> None: ... - class HelloRequest(_message.Message): - __slots__ = ["name"] + __slots__ = ("name",) NAME_FIELD_NUMBER: _ClassVar[int] name: str def __init__(self, name: _Optional[str] = ...) -> None: ... + +class HelloReply(_message.Message): + __slots__ = ("message",) + MESSAGE_FIELD_NUMBER: _ClassVar[int] + message: str + def __init__(self, message: _Optional[str] = ...) -> None: ... diff --git a/examples/python/helloworld/helloworld_pb2_grpc.py b/examples/python/helloworld/helloworld_pb2_grpc.py index 47c186976e1..68bcfef1756 100644 --- a/examples/python/helloworld/helloworld_pb2_grpc.py +++ b/examples/python/helloworld/helloworld_pb2_grpc.py @@ -19,7 +19,17 @@ class GreeterStub(object): '/helloworld.Greeter/SayHello', request_serializer=helloworld__pb2.HelloRequest.SerializeToString, response_deserializer=helloworld__pb2.HelloReply.FromString, - ) + _registered_method=True) + self.SayHelloStreamReply = channel.unary_stream( + '/helloworld.Greeter/SayHelloStreamReply', + request_serializer=helloworld__pb2.HelloRequest.SerializeToString, + response_deserializer=helloworld__pb2.HelloReply.FromString, + _registered_method=True) + self.SayHelloBidiStream = channel.stream_stream( + '/helloworld.Greeter/SayHelloBidiStream', + request_serializer=helloworld__pb2.HelloRequest.SerializeToString, + response_deserializer=helloworld__pb2.HelloReply.FromString, + _registered_method=True) class GreeterServicer(object): @@ -33,6 +43,18 @@ class GreeterServicer(object): context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') + def SayHelloStreamReply(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def SayHelloBidiStream(self, request_iterator, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + def add_GreeterServicer_to_server(servicer, server): rpc_method_handlers = { @@ -41,6 +63,16 @@ def add_GreeterServicer_to_server(servicer, server): request_deserializer=helloworld__pb2.HelloRequest.FromString, response_serializer=helloworld__pb2.HelloReply.SerializeToString, ), + 'SayHelloStreamReply': grpc.unary_stream_rpc_method_handler( + servicer.SayHelloStreamReply, + request_deserializer=helloworld__pb2.HelloRequest.FromString, + response_serializer=helloworld__pb2.HelloReply.SerializeToString, + ), + 'SayHelloBidiStream': grpc.stream_stream_rpc_method_handler( + servicer.SayHelloBidiStream, + request_deserializer=helloworld__pb2.HelloRequest.FromString, + response_serializer=helloworld__pb2.HelloReply.SerializeToString, + ), } generic_handler = grpc.method_handlers_generic_handler( 'helloworld.Greeter', rpc_method_handlers) @@ -63,8 +95,72 @@ class Greeter(object): wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/helloworld.Greeter/SayHello', + return grpc.experimental.unary_unary( + request, + target, + '/helloworld.Greeter/SayHello', + helloworld__pb2.HelloRequest.SerializeToString, + helloworld__pb2.HelloReply.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def SayHelloStreamReply(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream( + request, + target, + '/helloworld.Greeter/SayHelloStreamReply', + helloworld__pb2.HelloRequest.SerializeToString, + helloworld__pb2.HelloReply.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def SayHelloBidiStream(request_iterator, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.stream_stream( + request_iterator, + target, + '/helloworld.Greeter/SayHelloBidiStream', helloworld__pb2.HelloRequest.SerializeToString, helloworld__pb2.HelloReply.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/src/compiler/python_generator.cc b/src/compiler/python_generator.cc index 753fe1c8887..b7a3115bce0 100644 --- a/src/compiler/python_generator.cc +++ b/src/compiler/python_generator.cc @@ -467,7 +467,7 @@ bool PrivateGenerator::PrintStub( out->Print( method_dict, "response_deserializer=$ResponseModuleAndClass$.FromString,\n"); - out->Print(")\n"); + out->Print("_registered_method=True)\n"); } } } @@ -642,22 +642,27 @@ bool PrivateGenerator::PrintServiceClass( args_dict["ArityMethodName"] = arity_method_name; args_dict["PackageQualifiedService"] = package_qualified_service_name; args_dict["Method"] = method->name(); - out->Print(args_dict, - "return " - "grpc.experimental.$ArityMethodName$($RequestParameter$, " - "target, '/$PackageQualifiedService$/$Method$',\n"); + out->Print(args_dict, "return grpc.experimental.$ArityMethodName$(\n"); { IndentScope continuation_indent(out); StringMap serializer_dict; + out->Print(args_dict, "$RequestParameter$,\n"); + out->Print("target,\n"); + out->Print(args_dict, "'/$PackageQualifiedService$/$Method$',\n"); serializer_dict["RequestModuleAndClass"] = request_module_and_class; serializer_dict["ResponseModuleAndClass"] = response_module_and_class; out->Print(serializer_dict, "$RequestModuleAndClass$.SerializeToString,\n"); out->Print(serializer_dict, "$ResponseModuleAndClass$.FromString,\n"); - out->Print("options, channel_credentials,\n"); - out->Print( - "insecure, call_credentials, compression, wait_for_ready, " - "timeout, metadata)\n"); + out->Print("options,\n"); + out->Print("channel_credentials,\n"); + out->Print("insecure,\n"); + out->Print("call_credentials,\n"); + out->Print("compression,\n"); + out->Print("wait_for_ready,\n"); + out->Print("timeout,\n"); + out->Print("metadata,\n"); + out->Print("_registered_method=True)\n"); } } } diff --git a/src/python/.gitignore b/src/python/.gitignore index 095ab8bbae1..61363e8cb8d 100644 --- a/src/python/.gitignore +++ b/src/python/.gitignore @@ -1,4 +1,6 @@ -gens/ +build/ +grpc_root/ +third_party/ *_pb2.py *_pb2.pyi *_pb2_grpc.py diff --git a/src/python/grpcio/grpc/__init__.py b/src/python/grpcio/grpc/__init__.py index 83ded1b5df8..e0ec581f9d4 100644 --- a/src/python/grpcio/grpc/__init__.py +++ b/src/python/grpcio/grpc/__init__.py @@ -1004,6 +1004,7 @@ class Channel(abc.ABC): method, request_serializer=None, response_deserializer=None, + _registered_method=False, ): """Creates a UnaryUnaryMultiCallable for a unary-unary method. @@ -1014,6 +1015,8 @@ class Channel(abc.ABC): response_deserializer: Optional :term:`deserializer` for deserializing the response message. Response goes undeserialized in case None is passed. + _registered_method: Implementation Private. A bool representing whether the method + is registered. Returns: A UnaryUnaryMultiCallable value for the named unary-unary method. @@ -1026,6 +1029,7 @@ class Channel(abc.ABC): method, request_serializer=None, response_deserializer=None, + _registered_method=False, ): """Creates a UnaryStreamMultiCallable for a unary-stream method. @@ -1036,6 +1040,8 @@ class Channel(abc.ABC): response_deserializer: Optional :term:`deserializer` for deserializing the response message. Response goes undeserialized in case None is passed. + _registered_method: Implementation Private. A bool representing whether the method + is registered. Returns: A UnaryStreamMultiCallable value for the name unary-stream method. @@ -1048,6 +1054,7 @@ class Channel(abc.ABC): method, request_serializer=None, response_deserializer=None, + _registered_method=False, ): """Creates a StreamUnaryMultiCallable for a stream-unary method. @@ -1058,6 +1065,8 @@ class Channel(abc.ABC): response_deserializer: Optional :term:`deserializer` for deserializing the response message. Response goes undeserialized in case None is passed. + _registered_method: Implementation Private. A bool representing whether the method + is registered. Returns: A StreamUnaryMultiCallable value for the named stream-unary method. @@ -1070,6 +1079,7 @@ class Channel(abc.ABC): method, request_serializer=None, response_deserializer=None, + _registered_method=False, ): """Creates a StreamStreamMultiCallable for a stream-stream method. @@ -1080,6 +1090,8 @@ class Channel(abc.ABC): response_deserializer: Optional :term:`deserializer` for deserializing the response message. Response goes undeserialized in case None is passed. + _registered_method: Implementation Private. A bool representing whether the method + is registered. Returns: A StreamStreamMultiCallable value for the named stream-stream method. diff --git a/src/python/grpcio/grpc/_channel.py b/src/python/grpcio/grpc/_channel.py index 79c85a1b94c..bf29982ca65 100644 --- a/src/python/grpcio/grpc/_channel.py +++ b/src/python/grpcio/grpc/_channel.py @@ -24,6 +24,7 @@ import types from typing import ( Any, Callable, + Dict, Iterator, List, Optional, @@ -1054,6 +1055,7 @@ class _UnaryUnaryMultiCallable(grpc.UnaryUnaryMultiCallable): _request_serializer: Optional[SerializingFunction] _response_deserializer: Optional[DeserializingFunction] _context: Any + _registered_call_handle: Optional[int] __slots__ = [ "_channel", @@ -1074,6 +1076,7 @@ class _UnaryUnaryMultiCallable(grpc.UnaryUnaryMultiCallable): target: bytes, request_serializer: Optional[SerializingFunction], response_deserializer: Optional[DeserializingFunction], + _registered_call_handle: Optional[int], ): self._channel = channel self._managed_call = managed_call @@ -1082,6 +1085,7 @@ class _UnaryUnaryMultiCallable(grpc.UnaryUnaryMultiCallable): self._request_serializer = request_serializer self._response_deserializer = response_deserializer self._context = cygrpc.build_census_context() + self._registered_call_handle = _registered_call_handle def _prepare( self, @@ -1153,6 +1157,7 @@ class _UnaryUnaryMultiCallable(grpc.UnaryUnaryMultiCallable): ), ), self._context, + self._registered_call_handle, ) event = call.next_event() _handle_event(event, state, self._response_deserializer) @@ -1221,6 +1226,7 @@ class _UnaryUnaryMultiCallable(grpc.UnaryUnaryMultiCallable): (operations,), event_handler, self._context, + self._registered_call_handle, ) return _MultiThreadedRendezvous( state, call, self._response_deserializer, deadline @@ -1234,6 +1240,7 @@ class _SingleThreadedUnaryStreamMultiCallable(grpc.UnaryStreamMultiCallable): _request_serializer: Optional[SerializingFunction] _response_deserializer: Optional[DeserializingFunction] _context: Any + _registered_call_handle: Optional[int] __slots__ = [ "_channel", @@ -1252,6 +1259,7 @@ class _SingleThreadedUnaryStreamMultiCallable(grpc.UnaryStreamMultiCallable): target: bytes, request_serializer: SerializingFunction, response_deserializer: DeserializingFunction, + _registered_call_handle: Optional[int], ): self._channel = channel self._method = method @@ -1259,6 +1267,7 @@ class _SingleThreadedUnaryStreamMultiCallable(grpc.UnaryStreamMultiCallable): self._request_serializer = request_serializer self._response_deserializer = response_deserializer self._context = cygrpc.build_census_context() + self._registered_call_handle = _registered_call_handle def __call__( # pylint: disable=too-many-locals self, @@ -1317,6 +1326,7 @@ class _SingleThreadedUnaryStreamMultiCallable(grpc.UnaryStreamMultiCallable): call_credentials, operations_and_tags, self._context, + self._registered_call_handle, ) return _SingleThreadedRendezvous( state, call, self._response_deserializer, deadline @@ -1331,6 +1341,7 @@ class _UnaryStreamMultiCallable(grpc.UnaryStreamMultiCallable): _request_serializer: Optional[SerializingFunction] _response_deserializer: Optional[DeserializingFunction] _context: Any + _registered_call_handle: Optional[int] __slots__ = [ "_channel", @@ -1351,6 +1362,7 @@ class _UnaryStreamMultiCallable(grpc.UnaryStreamMultiCallable): target: bytes, request_serializer: SerializingFunction, response_deserializer: DeserializingFunction, + _registered_call_handle: Optional[int], ): self._channel = channel self._managed_call = managed_call @@ -1359,6 +1371,7 @@ class _UnaryStreamMultiCallable(grpc.UnaryStreamMultiCallable): self._request_serializer = request_serializer self._response_deserializer = response_deserializer self._context = cygrpc.build_census_context() + self._registered_call_handle = _registered_call_handle def __call__( # pylint: disable=too-many-locals self, @@ -1408,6 +1421,7 @@ class _UnaryStreamMultiCallable(grpc.UnaryStreamMultiCallable): operations, _event_handler(state, self._response_deserializer), self._context, + self._registered_call_handle, ) return _MultiThreadedRendezvous( state, call, self._response_deserializer, deadline @@ -1422,6 +1436,7 @@ class _StreamUnaryMultiCallable(grpc.StreamUnaryMultiCallable): _request_serializer: Optional[SerializingFunction] _response_deserializer: Optional[DeserializingFunction] _context: Any + _registered_call_handle: Optional[int] __slots__ = [ "_channel", @@ -1442,6 +1457,7 @@ class _StreamUnaryMultiCallable(grpc.StreamUnaryMultiCallable): target: bytes, request_serializer: Optional[SerializingFunction], response_deserializer: Optional[DeserializingFunction], + _registered_call_handle: Optional[int], ): self._channel = channel self._managed_call = managed_call @@ -1450,6 +1466,7 @@ class _StreamUnaryMultiCallable(grpc.StreamUnaryMultiCallable): self._request_serializer = request_serializer self._response_deserializer = response_deserializer self._context = cygrpc.build_census_context() + self._registered_call_handle = _registered_call_handle def _blocking( self, @@ -1482,6 +1499,7 @@ class _StreamUnaryMultiCallable(grpc.StreamUnaryMultiCallable): augmented_metadata, initial_metadata_flags ), self._context, + self._registered_call_handle, ) _consume_request_iterator( request_iterator, state, call, self._request_serializer, None @@ -1572,6 +1590,7 @@ class _StreamUnaryMultiCallable(grpc.StreamUnaryMultiCallable): ), event_handler, self._context, + self._registered_call_handle, ) _consume_request_iterator( request_iterator, @@ -1593,6 +1612,7 @@ class _StreamStreamMultiCallable(grpc.StreamStreamMultiCallable): _request_serializer: Optional[SerializingFunction] _response_deserializer: Optional[DeserializingFunction] _context: Any + _registered_call_handle: Optional[int] __slots__ = [ "_channel", @@ -1611,8 +1631,9 @@ class _StreamStreamMultiCallable(grpc.StreamStreamMultiCallable): managed_call: IntegratedCallFactory, method: bytes, target: bytes, - request_serializer: Optional[SerializingFunction] = None, - response_deserializer: Optional[DeserializingFunction] = None, + request_serializer: Optional[SerializingFunction], + response_deserializer: Optional[DeserializingFunction], + _registered_call_handle: Optional[int], ): self._channel = channel self._managed_call = managed_call @@ -1621,6 +1642,7 @@ class _StreamStreamMultiCallable(grpc.StreamStreamMultiCallable): self._request_serializer = request_serializer self._response_deserializer = response_deserializer self._context = cygrpc.build_census_context() + self._registered_call_handle = _registered_call_handle def __call__( self, @@ -1662,6 +1684,7 @@ class _StreamStreamMultiCallable(grpc.StreamStreamMultiCallable): operations, event_handler, self._context, + self._registered_call_handle, ) _consume_request_iterator( request_iterator, @@ -1751,7 +1774,8 @@ def _channel_managed_call_management(state: _ChannelCallState): credentials: Optional[cygrpc.CallCredentials], operations: Sequence[Sequence[cygrpc.Operation]], event_handler: UserTag, - context, + context: Any, + _registered_call_handle: Optional[int], ) -> cygrpc.IntegratedCall: """Creates a cygrpc.IntegratedCall. @@ -1768,6 +1792,8 @@ def _channel_managed_call_management(state: _ChannelCallState): event_handler: A behavior to call to handle the events resultant from the operations on the call. context: Context object for distributed tracing. + _registered_call_handle: An int representing the call handle of the + method, or None if the method is not registered. Returns: A cygrpc.IntegratedCall with which to conduct an RPC. """ @@ -1788,6 +1814,7 @@ def _channel_managed_call_management(state: _ChannelCallState): credentials, operations_and_tags, context, + _registered_call_handle, ) if state.managed_calls == 0: state.managed_calls = 1 @@ -2021,6 +2048,7 @@ class Channel(grpc.Channel): _call_state: _ChannelCallState _connectivity_state: _ChannelConnectivityState _target: str + _registered_call_handles: Dict[str, int] def __init__( self, @@ -2055,6 +2083,22 @@ class Channel(grpc.Channel): if cygrpc.g_gevent_activated: cygrpc.gevent_increment_channel_count() + def _get_registered_call_handle(self, method: str) -> int: + """ + Get the registered call handle for a method. + + This is a semi-private method. It is intended for use only by gRPC generated code. + + This method is not thread-safe. + + Args: + method: Required, the method name for the RPC. + + Returns: + The registered call handle pointer in the form of a Python Long. + """ + return self._channel.get_registered_call_handle(_common.encode(method)) + def _process_python_options( self, python_options: Sequence[ChannelArgumentType] ) -> None: @@ -2078,12 +2122,17 @@ class Channel(grpc.Channel): ) -> None: _unsubscribe(self._connectivity_state, callback) + # pylint: disable=arguments-differ def unary_unary( self, method: str, request_serializer: Optional[SerializingFunction] = None, response_deserializer: Optional[DeserializingFunction] = None, + _registered_method: Optional[bool] = False, ) -> grpc.UnaryUnaryMultiCallable: + _registered_call_handle = None + if _registered_method: + _registered_call_handle = self._get_registered_call_handle(method) return _UnaryUnaryMultiCallable( self._channel, _channel_managed_call_management(self._call_state), @@ -2091,14 +2140,20 @@ class Channel(grpc.Channel): _common.encode(self._target), request_serializer, response_deserializer, + _registered_call_handle, ) + # pylint: disable=arguments-differ def unary_stream( self, method: str, request_serializer: Optional[SerializingFunction] = None, response_deserializer: Optional[DeserializingFunction] = None, + _registered_method: Optional[bool] = False, ) -> grpc.UnaryStreamMultiCallable: + _registered_call_handle = None + if _registered_method: + _registered_call_handle = self._get_registered_call_handle(method) # NOTE(rbellevi): Benchmarks have shown that running a unary-stream RPC # on a single Python thread results in an appreciable speed-up. However, # due to slight differences in capability, the multi-threaded variant @@ -2110,6 +2165,7 @@ class Channel(grpc.Channel): _common.encode(self._target), request_serializer, response_deserializer, + _registered_call_handle, ) else: return _UnaryStreamMultiCallable( @@ -2119,14 +2175,20 @@ class Channel(grpc.Channel): _common.encode(self._target), request_serializer, response_deserializer, + _registered_call_handle, ) + # pylint: disable=arguments-differ def stream_unary( self, method: str, request_serializer: Optional[SerializingFunction] = None, response_deserializer: Optional[DeserializingFunction] = None, + _registered_method: Optional[bool] = False, ) -> grpc.StreamUnaryMultiCallable: + _registered_call_handle = None + if _registered_method: + _registered_call_handle = self._get_registered_call_handle(method) return _StreamUnaryMultiCallable( self._channel, _channel_managed_call_management(self._call_state), @@ -2134,14 +2196,20 @@ class Channel(grpc.Channel): _common.encode(self._target), request_serializer, response_deserializer, + _registered_call_handle, ) + # pylint: disable=arguments-differ def stream_stream( self, method: str, request_serializer: Optional[SerializingFunction] = None, response_deserializer: Optional[DeserializingFunction] = None, + _registered_method: Optional[bool] = False, ) -> grpc.StreamStreamMultiCallable: + _registered_call_handle = None + if _registered_method: + _registered_call_handle = self._get_registered_call_handle(method) return _StreamStreamMultiCallable( self._channel, _channel_managed_call_management(self._call_state), @@ -2149,6 +2217,7 @@ class Channel(grpc.Channel): _common.encode(self._target), request_serializer, response_deserializer, + _registered_call_handle, ) def _unsubscribe_all(self) -> None: diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/channel.pxd.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/channel.pxd.pxi index 6e5416a9e31..96d03e181b9 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/channel.pxd.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/channel.pxd.pxi @@ -74,6 +74,13 @@ cdef class SegregatedCall: cdef class Channel: cdef _ChannelState _state + cdef dict _registered_call_handles # TODO(https://github.com/grpc/grpc/issues/15662): Eliminate this. cdef tuple _arguments + + +cdef class CallHandle: + + cdef void *c_call_handle + cdef object method diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi index f6db36ebde1..dde3b166789 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi @@ -101,6 +101,25 @@ cdef class _ChannelState: self.connectivity_due = set() self.closed_reason = None +cdef class CallHandle: + + def __cinit__(self, _ChannelState channel_state, object method): + self.method = method + cpython.Py_INCREF(method) + # Note that since we always pass None for host, we set the + # second-to-last parameter of grpc_channel_register_call to a fixed + # NULL value. + self.c_call_handle = grpc_channel_register_call( + channel_state.c_channel, method, NULL, NULL) + + def __dealloc__(self): + cpython.Py_DECREF(self.method) + + @property + def call_handle(self): + return cpython.PyLong_FromVoidPtr(self.c_call_handle) + + cdef tuple _operate(grpc_call *c_call, object operations, object user_tag): cdef grpc_call_error c_call_error @@ -199,7 +218,7 @@ cdef void _call( grpc_completion_queue *c_completion_queue, on_success, int flags, method, host, object deadline, CallCredentials credentials, object operationses_and_user_tags, object metadata, - object context) except *: + object context, object registered_call_handle) except *: """Invokes an RPC. Args: @@ -226,6 +245,8 @@ cdef void _call( must be present in the first element of this value. metadata: The metadata for this call. context: Context object for distributed tracing. + registered_call_handle: An int representing the call handle of the method, or + None if the method is not registered. """ cdef grpc_slice method_slice cdef grpc_slice host_slice @@ -242,10 +263,16 @@ cdef void _call( else: host_slice = _slice_from_bytes(host) host_slice_ptr = &host_slice - call_state.c_call = grpc_channel_create_call( - channel_state.c_channel, NULL, flags, - c_completion_queue, method_slice, host_slice_ptr, - _timespec_from_time(deadline), NULL) + if registered_call_handle: + call_state.c_call = grpc_channel_create_registered_call( + channel_state.c_channel, NULL, flags, + c_completion_queue, cpython.PyLong_AsVoidPtr(registered_call_handle), + _timespec_from_time(deadline), NULL) + else: + call_state.c_call = grpc_channel_create_call( + channel_state.c_channel, NULL, flags, + c_completion_queue, method_slice, host_slice_ptr, + _timespec_from_time(deadline), NULL) grpc_slice_unref(method_slice) if host_slice_ptr: grpc_slice_unref(host_slice) @@ -309,7 +336,7 @@ cdef class IntegratedCall: cdef IntegratedCall _integrated_call( _ChannelState state, int flags, method, host, object deadline, object metadata, CallCredentials credentials, operationses_and_user_tags, - object context): + object context, object registered_call_handle): call_state = _CallState() def on_success(started_tags): @@ -318,7 +345,8 @@ cdef IntegratedCall _integrated_call( _call( state, call_state, state.c_call_completion_queue, on_success, flags, - method, host, deadline, credentials, operationses_and_user_tags, metadata, context) + method, host, deadline, credentials, operationses_and_user_tags, + metadata, context, registered_call_handle) return IntegratedCall(state, call_state) @@ -371,7 +399,7 @@ cdef class SegregatedCall: cdef SegregatedCall _segregated_call( _ChannelState state, int flags, method, host, object deadline, object metadata, CallCredentials credentials, operationses_and_user_tags, - object context): + object context, object registered_call_handle): cdef _CallState call_state = _CallState() cdef SegregatedCall segregated_call cdef grpc_completion_queue *c_completion_queue @@ -389,7 +417,7 @@ cdef SegregatedCall _segregated_call( _call( state, call_state, c_completion_queue, on_success, flags, method, host, deadline, credentials, operationses_and_user_tags, metadata, - context) + context, registered_call_handle) except: _destroy_c_completion_queue(c_completion_queue) raise @@ -486,6 +514,7 @@ cdef class Channel: else grpc_insecure_credentials_create()) self._state.c_channel = grpc_channel_create( target, c_channel_credentials, channel_args.c_args()) + self._registered_call_handles = {} grpc_channel_credentials_release(c_channel_credentials) def target(self): @@ -499,10 +528,10 @@ cdef class Channel: def integrated_call( self, int flags, method, host, object deadline, object metadata, CallCredentials credentials, operationses_and_tags, - object context = None): + object context = None, object registered_call_handle = None): return _integrated_call( self._state, flags, method, host, deadline, metadata, credentials, - operationses_and_tags, context) + operationses_and_tags, context, registered_call_handle) def next_call_event(self): def on_success(tag): @@ -521,10 +550,10 @@ cdef class Channel: def segregated_call( self, int flags, method, host, object deadline, object metadata, CallCredentials credentials, operationses_and_tags, - object context = None): + object context = None, object registered_call_handle = None): return _segregated_call( self._state, flags, method, host, deadline, metadata, credentials, - operationses_and_tags, context) + operationses_and_tags, context, registered_call_handle) def check_connectivity_state(self, bint try_to_connect): with self._state.condition: @@ -543,3 +572,19 @@ cdef class Channel: def close_on_fork(self, code, details): _close(self, code, details, True) + + def get_registered_call_handle(self, method): + """ + Get or registers a call handler for a method. + + This method is not thread-safe. + + Args: + method: Required, the method name for the RPC. + + Returns: + The registered call handle pointer in the form of a Python Long. + """ + if method not in self._registered_call_handles.keys(): + self._registered_call_handles[method] = CallHandle(self._state, method) + return self._registered_call_handles[method].call_handle diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi index e1bc87d4abd..29149e9893a 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi @@ -433,6 +433,12 @@ cdef extern from "grpc/grpc.h": grpc_channel *channel, grpc_call *parent_call, uint32_t propagation_mask, grpc_completion_queue *completion_queue, grpc_slice method, const grpc_slice *host, gpr_timespec deadline, void *reserved) nogil + void *grpc_channel_register_call( + grpc_channel *channel, const char *method, const char *host, void *reserved) nogil + grpc_call *grpc_channel_create_registered_call( + grpc_channel *channel, grpc_call *parent_call, uint32_t propagation_mask, + grpc_completion_queue *completion_queue, void* registered_call_handle, + gpr_timespec deadline, void *reserved) nogil grpc_connectivity_state grpc_channel_check_connectivity_state( grpc_channel *channel, int try_to_connect) nogil void grpc_channel_watch_connectivity_state( diff --git a/src/python/grpcio/grpc/_interceptor.py b/src/python/grpcio/grpc/_interceptor.py index 36bce4e3ba5..94abafebaa6 100644 --- a/src/python/grpcio/grpc/_interceptor.py +++ b/src/python/grpcio/grpc/_interceptor.py @@ -684,57 +684,85 @@ class _Channel(grpc.Channel): def unsubscribe(self, callback: Callable): self._channel.unsubscribe(callback) + # pylint: disable=arguments-differ def unary_unary( self, method: str, request_serializer: Optional[SerializingFunction] = None, response_deserializer: Optional[DeserializingFunction] = None, + _registered_method: Optional[bool] = False, ) -> grpc.UnaryUnaryMultiCallable: + # pytype: disable=wrong-arg-count thunk = lambda m: self._channel.unary_unary( - m, request_serializer, response_deserializer + m, + request_serializer, + response_deserializer, + _registered_method, ) + # pytype: enable=wrong-arg-count if isinstance(self._interceptor, grpc.UnaryUnaryClientInterceptor): return _UnaryUnaryMultiCallable(thunk, method, self._interceptor) else: return thunk(method) + # pylint: disable=arguments-differ def unary_stream( self, method: str, request_serializer: Optional[SerializingFunction] = None, response_deserializer: Optional[DeserializingFunction] = None, + _registered_method: Optional[bool] = False, ) -> grpc.UnaryStreamMultiCallable: + # pytype: disable=wrong-arg-count thunk = lambda m: self._channel.unary_stream( - m, request_serializer, response_deserializer + m, + request_serializer, + response_deserializer, + _registered_method, ) + # pytype: enable=wrong-arg-count if isinstance(self._interceptor, grpc.UnaryStreamClientInterceptor): return _UnaryStreamMultiCallable(thunk, method, self._interceptor) else: return thunk(method) + # pylint: disable=arguments-differ def stream_unary( self, method: str, request_serializer: Optional[SerializingFunction] = None, response_deserializer: Optional[DeserializingFunction] = None, + _registered_method: Optional[bool] = False, ) -> grpc.StreamUnaryMultiCallable: + # pytype: disable=wrong-arg-count thunk = lambda m: self._channel.stream_unary( - m, request_serializer, response_deserializer + m, + request_serializer, + response_deserializer, + _registered_method, ) + # pytype: enable=wrong-arg-count if isinstance(self._interceptor, grpc.StreamUnaryClientInterceptor): return _StreamUnaryMultiCallable(thunk, method, self._interceptor) else: return thunk(method) + # pylint: disable=arguments-differ def stream_stream( self, method: str, request_serializer: Optional[SerializingFunction] = None, response_deserializer: Optional[DeserializingFunction] = None, + _registered_method: Optional[bool] = False, ) -> grpc.StreamStreamMultiCallable: + # pytype: disable=wrong-arg-count thunk = lambda m: self._channel.stream_stream( - m, request_serializer, response_deserializer + m, + request_serializer, + response_deserializer, + _registered_method, ) + # pytype: enable=wrong-arg-count if isinstance(self._interceptor, grpc.StreamStreamClientInterceptor): return _StreamStreamMultiCallable(thunk, method, self._interceptor) else: diff --git a/src/python/grpcio/grpc/_simple_stubs.py b/src/python/grpcio/grpc/_simple_stubs.py index 7772860957b..3e88670aa08 100644 --- a/src/python/grpcio/grpc/_simple_stubs.py +++ b/src/python/grpcio/grpc/_simple_stubs.py @@ -159,7 +159,19 @@ class ChannelCache: channel_credentials: Optional[grpc.ChannelCredentials], insecure: bool, compression: Optional[grpc.Compression], - ) -> grpc.Channel: + method: str, + _registered_method: bool, + ) -> Tuple[grpc.Channel, Optional[int]]: + """Get a channel from cache or creates a new channel. + + This method also takes care of register method for channel, + which means we'll register a new call handle if we're calling a + non-registered method for an existing channel. + + Returns: + A tuple with two items. The first item is the channel, second item is + the call handle if the method is registered, None if it's not registered. + """ if insecure and channel_credentials: raise ValueError( "The insecure option is mutually exclusive with " @@ -176,18 +188,25 @@ class ChannelCache: key = (target, options, channel_credentials, compression) with self._lock: channel_data = self._mapping.get(key, None) + call_handle = None if channel_data is not None: channel = channel_data[0] + # Register a new call handle if we're calling a registered method for an + # existing channel and this method is not registered. + if _registered_method: + call_handle = channel._get_registered_call_handle(method) self._mapping.pop(key) self._mapping[key] = ( channel, datetime.datetime.now() + _EVICTION_PERIOD, ) - return channel + return channel, call_handle else: channel = _create_channel( target, options, channel_credentials, compression ) + if _registered_method: + call_handle = channel._get_registered_call_handle(method) self._mapping[key] = ( channel, datetime.datetime.now() + _EVICTION_PERIOD, @@ -197,7 +216,7 @@ class ChannelCache: or len(self._mapping) >= _MAXIMUM_CHANNELS ): self._condition.notify() - return channel + return channel, call_handle def _test_only_channel_count(self) -> int: with self._lock: @@ -205,6 +224,7 @@ class ChannelCache: @experimental_api +# pylint: disable=too-many-locals def unary_unary( request: RequestType, target: str, @@ -219,6 +239,7 @@ def unary_unary( wait_for_ready: Optional[bool] = None, timeout: Optional[float] = _DEFAULT_TIMEOUT, metadata: Optional[Sequence[Tuple[str, Union[str, bytes]]]] = None, + _registered_method: Optional[bool] = False, ) -> ResponseType: """Invokes a unary-unary RPC without an explicitly specified channel. @@ -272,11 +293,17 @@ def unary_unary( Returns: The response to the RPC. """ - channel = ChannelCache.get().get_channel( - target, options, channel_credentials, insecure, compression + channel, method_handle = ChannelCache.get().get_channel( + target, + options, + channel_credentials, + insecure, + compression, + method, + _registered_method, ) multicallable = channel.unary_unary( - method, request_serializer, response_deserializer + method, request_serializer, response_deserializer, method_handle ) wait_for_ready = wait_for_ready if wait_for_ready is not None else True return multicallable( @@ -289,6 +316,7 @@ def unary_unary( @experimental_api +# pylint: disable=too-many-locals def unary_stream( request: RequestType, target: str, @@ -303,6 +331,7 @@ def unary_stream( wait_for_ready: Optional[bool] = None, timeout: Optional[float] = _DEFAULT_TIMEOUT, metadata: Optional[Sequence[Tuple[str, Union[str, bytes]]]] = None, + _registered_method: Optional[bool] = False, ) -> Iterator[ResponseType]: """Invokes a unary-stream RPC without an explicitly specified channel. @@ -355,11 +384,17 @@ def unary_stream( Returns: An iterator of responses. """ - channel = ChannelCache.get().get_channel( - target, options, channel_credentials, insecure, compression + channel, method_handle = ChannelCache.get().get_channel( + target, + options, + channel_credentials, + insecure, + compression, + method, + _registered_method, ) multicallable = channel.unary_stream( - method, request_serializer, response_deserializer + method, request_serializer, response_deserializer, method_handle ) wait_for_ready = wait_for_ready if wait_for_ready is not None else True return multicallable( @@ -372,6 +407,7 @@ def unary_stream( @experimental_api +# pylint: disable=too-many-locals def stream_unary( request_iterator: Iterator[RequestType], target: str, @@ -386,6 +422,7 @@ def stream_unary( wait_for_ready: Optional[bool] = None, timeout: Optional[float] = _DEFAULT_TIMEOUT, metadata: Optional[Sequence[Tuple[str, Union[str, bytes]]]] = None, + _registered_method: Optional[bool] = False, ) -> ResponseType: """Invokes a stream-unary RPC without an explicitly specified channel. @@ -438,11 +475,17 @@ def stream_unary( Returns: The response to the RPC. """ - channel = ChannelCache.get().get_channel( - target, options, channel_credentials, insecure, compression + channel, method_handle = ChannelCache.get().get_channel( + target, + options, + channel_credentials, + insecure, + compression, + method, + _registered_method, ) multicallable = channel.stream_unary( - method, request_serializer, response_deserializer + method, request_serializer, response_deserializer, method_handle ) wait_for_ready = wait_for_ready if wait_for_ready is not None else True return multicallable( @@ -455,6 +498,7 @@ def stream_unary( @experimental_api +# pylint: disable=too-many-locals def stream_stream( request_iterator: Iterator[RequestType], target: str, @@ -469,6 +513,7 @@ def stream_stream( wait_for_ready: Optional[bool] = None, timeout: Optional[float] = _DEFAULT_TIMEOUT, metadata: Optional[Sequence[Tuple[str, Union[str, bytes]]]] = None, + _registered_method: Optional[bool] = False, ) -> Iterator[ResponseType]: """Invokes a stream-stream RPC without an explicitly specified channel. @@ -521,11 +566,17 @@ def stream_stream( Returns: An iterator of responses. """ - channel = ChannelCache.get().get_channel( - target, options, channel_credentials, insecure, compression + channel, method_handle = ChannelCache.get().get_channel( + target, + options, + channel_credentials, + insecure, + compression, + method, + _registered_method, ) multicallable = channel.stream_stream( - method, request_serializer, response_deserializer + method, request_serializer, response_deserializer, method_handle ) wait_for_ready = wait_for_ready if wait_for_ready is not None else True return multicallable( diff --git a/src/python/grpcio/grpc/aio/_base_channel.py b/src/python/grpcio/grpc/aio/_base_channel.py index 2fb8e75d3a9..8bc32b8b641 100644 --- a/src/python/grpcio/grpc/aio/_base_channel.py +++ b/src/python/grpcio/grpc/aio/_base_channel.py @@ -272,6 +272,7 @@ class Channel(abc.ABC): method: str, request_serializer: Optional[SerializingFunction] = None, response_deserializer: Optional[DeserializingFunction] = None, + _registered_method: Optional[bool] = False, ) -> UnaryUnaryMultiCallable: """Creates a UnaryUnaryMultiCallable for a unary-unary method. @@ -282,6 +283,8 @@ class Channel(abc.ABC): response_deserializer: Optional :term:`deserializer` for deserializing the response message. Response goes undeserialized in case None is passed. + _registered_method: Implementation Private. Optional: A bool representing + whether the method is registered. Returns: A UnaryUnaryMultiCallable value for the named unary-unary method. @@ -293,6 +296,7 @@ class Channel(abc.ABC): method: str, request_serializer: Optional[SerializingFunction] = None, response_deserializer: Optional[DeserializingFunction] = None, + _registered_method: Optional[bool] = False, ) -> UnaryStreamMultiCallable: """Creates a UnaryStreamMultiCallable for a unary-stream method. @@ -303,6 +307,8 @@ class Channel(abc.ABC): response_deserializer: Optional :term:`deserializer` for deserializing the response message. Response goes undeserialized in case None is passed. + _registered_method: Implementation Private. Optional: A bool representing + whether the method is registered. Returns: A UnarySteramMultiCallable value for the named unary-stream method. @@ -314,6 +320,7 @@ class Channel(abc.ABC): method: str, request_serializer: Optional[SerializingFunction] = None, response_deserializer: Optional[DeserializingFunction] = None, + _registered_method: Optional[bool] = False, ) -> StreamUnaryMultiCallable: """Creates a StreamUnaryMultiCallable for a stream-unary method. @@ -324,6 +331,8 @@ class Channel(abc.ABC): response_deserializer: Optional :term:`deserializer` for deserializing the response message. Response goes undeserialized in case None is passed. + _registered_method: Implementation Private. Optional: A bool representing + whether the method is registered. Returns: A StreamUnaryMultiCallable value for the named stream-unary method. @@ -335,6 +344,7 @@ class Channel(abc.ABC): method: str, request_serializer: Optional[SerializingFunction] = None, response_deserializer: Optional[DeserializingFunction] = None, + _registered_method: Optional[bool] = False, ) -> StreamStreamMultiCallable: """Creates a StreamStreamMultiCallable for a stream-stream method. @@ -345,6 +355,8 @@ class Channel(abc.ABC): response_deserializer: Optional :term:`deserializer` for deserializing the response message. Response goes undeserialized in case None is passed. + _registered_method: Implementation Private. Optional: A bool representing + whether the method is registered. Returns: A StreamStreamMultiCallable value for the named stream-stream method. diff --git a/src/python/grpcio/grpc/aio/_channel.py b/src/python/grpcio/grpc/aio/_channel.py index bea64c27fa3..ea4de20965a 100644 --- a/src/python/grpcio/grpc/aio/_channel.py +++ b/src/python/grpcio/grpc/aio/_channel.py @@ -478,11 +478,20 @@ class Channel(_base_channel.Channel): await self.wait_for_state_change(state) state = self.get_state(try_to_connect=True) + # TODO(xuanwn): Implement this method after we have + # observability for Asyncio. + def _get_registered_call_handle(self, method: str) -> int: + pass + + # TODO(xuanwn): Implement _registered_method after we have + # observability for Asyncio. + # pylint: disable=arguments-differ,unused-argument def unary_unary( self, method: str, request_serializer: Optional[SerializingFunction] = None, response_deserializer: Optional[DeserializingFunction] = None, + _registered_method: Optional[bool] = False, ) -> UnaryUnaryMultiCallable: return UnaryUnaryMultiCallable( self._channel, @@ -494,11 +503,15 @@ class Channel(_base_channel.Channel): self._loop, ) + # TODO(xuanwn): Implement _registered_method after we have + # observability for Asyncio. + # pylint: disable=arguments-differ,unused-argument def unary_stream( self, method: str, request_serializer: Optional[SerializingFunction] = None, response_deserializer: Optional[DeserializingFunction] = None, + _registered_method: Optional[bool] = False, ) -> UnaryStreamMultiCallable: return UnaryStreamMultiCallable( self._channel, @@ -510,11 +523,15 @@ class Channel(_base_channel.Channel): self._loop, ) + # TODO(xuanwn): Implement _registered_method after we have + # observability for Asyncio. + # pylint: disable=arguments-differ,unused-argument def stream_unary( self, method: str, request_serializer: Optional[SerializingFunction] = None, response_deserializer: Optional[DeserializingFunction] = None, + _registered_method: Optional[bool] = False, ) -> StreamUnaryMultiCallable: return StreamUnaryMultiCallable( self._channel, @@ -526,11 +543,15 @@ class Channel(_base_channel.Channel): self._loop, ) + # TODO(xuanwn): Implement _registered_method after we have + # observability for Asyncio. + # pylint: disable=arguments-differ,unused-argument def stream_stream( self, method: str, request_serializer: Optional[SerializingFunction] = None, response_deserializer: Optional[DeserializingFunction] = None, + _registered_method: Optional[bool] = False, ) -> StreamStreamMultiCallable: return StreamStreamMultiCallable( self._channel, diff --git a/src/python/grpcio_testing/grpc_testing/_channel/_channel.py b/src/python/grpcio_testing/grpc_testing/_channel/_channel.py index 170533f63ea..3f12e1f4df8 100644 --- a/src/python/grpcio_testing/grpc_testing/_channel/_channel.py +++ b/src/python/grpcio_testing/grpc_testing/_channel/_channel.py @@ -31,23 +31,42 @@ class TestingChannel(grpc_testing.Channel): def unsubscribe(self, callback): raise NotImplementedError() + def _get_registered_call_handle(self, method: str) -> int: + pass + def unary_unary( - self, method, request_serializer=None, response_deserializer=None + self, + method, + request_serializer=None, + response_deserializer=None, + _registered_method=False, ): return _multi_callable.UnaryUnary(method, self._state) def unary_stream( - self, method, request_serializer=None, response_deserializer=None + self, + method, + request_serializer=None, + response_deserializer=None, + _registered_method=False, ): return _multi_callable.UnaryStream(method, self._state) def stream_unary( - self, method, request_serializer=None, response_deserializer=None + self, + method, + request_serializer=None, + response_deserializer=None, + _registered_method=False, ): return _multi_callable.StreamUnary(method, self._state) def stream_stream( - self, method, request_serializer=None, response_deserializer=None + self, + method, + request_serializer=None, + response_deserializer=None, + _registered_method=False, ): return _multi_callable.StreamStream(method, self._state) diff --git a/src/python/grpcio_tests/tests/channelz/_channelz_servicer_test.py b/src/python/grpcio_tests/tests/channelz/_channelz_servicer_test.py index 78333fc62c7..2379ab59a3c 100644 --- a/src/python/grpcio_tests/tests/channelz/_channelz_servicer_test.py +++ b/src/python/grpcio_tests/tests/channelz/_channelz_servicer_test.py @@ -99,16 +99,20 @@ class ChannelzServicerTest(unittest.TestCase): def _send_successful_unary_unary(self, idx): _, r = ( self._pairs[idx] - .channel.unary_unary(_SUCCESSFUL_UNARY_UNARY) + .channel.unary_unary( + _SUCCESSFUL_UNARY_UNARY, + _registered_method=True, + ) .with_call(_REQUEST) ) self.assertEqual(r.code(), grpc.StatusCode.OK) def _send_failed_unary_unary(self, idx): try: - self._pairs[idx].channel.unary_unary(_FAILED_UNARY_UNARY).with_call( - _REQUEST - ) + self._pairs[idx].channel.unary_unary( + _FAILED_UNARY_UNARY, + _registered_method=True, + ).with_call(_REQUEST) except grpc.RpcError: return else: @@ -117,7 +121,10 @@ class ChannelzServicerTest(unittest.TestCase): def _send_successful_stream_stream(self, idx): response_iterator = ( self._pairs[idx] - .channel.stream_stream(_SUCCESSFUL_STREAM_STREAM) + .channel.stream_stream( + _SUCCESSFUL_STREAM_STREAM, + _registered_method=True, + ) .__call__(iter([_REQUEST] * test_constants.STREAM_LENGTH)) ) cnt = 0 diff --git a/src/python/grpcio_tests/tests/csds/csds_test.py b/src/python/grpcio_tests/tests/csds/csds_test.py index 365115bcb99..5a882b2d679 100644 --- a/src/python/grpcio_tests/tests/csds/csds_test.py +++ b/src/python/grpcio_tests/tests/csds/csds_test.py @@ -86,7 +86,10 @@ class TestCsds(unittest.TestCase): # Force the XdsClient to initialize and request a resource with self.assertRaises(grpc.RpcError) as rpc_error: - dummy_channel.unary_unary("")(b"", wait_for_ready=False, timeout=1) + dummy_channel.unary_unary( + "", + _registered_method=True, + )(b"", wait_for_ready=False, timeout=1) self.assertEqual( grpc.StatusCode.DEADLINE_EXCEEDED, rpc_error.exception.code() ) diff --git a/src/python/grpcio_tests/tests/protoc_plugin/_python_plugin_test.py b/src/python/grpcio_tests/tests/protoc_plugin/_python_plugin_test.py index 9a6538cca35..c6e41b3ae8c 100644 --- a/src/python/grpcio_tests/tests/protoc_plugin/_python_plugin_test.py +++ b/src/python/grpcio_tests/tests/protoc_plugin/_python_plugin_test.py @@ -543,6 +543,17 @@ class PythonPluginTest(unittest.TestCase): ) service.server.stop(None) + def testRegisteredMethod(self): + """Tests that we're setting _registered_call_handle when create call using generated stub.""" + service = _CreateService() + self.assertTrue(service.stub.UnaryCall._registered_call_handle) + self.assertTrue( + service.stub.StreamingOutputCall._registered_call_handle + ) + self.assertTrue(service.stub.StreamingInputCall._registered_call_handle) + self.assertTrue(service.stub.FullDuplexCall._registered_call_handle) + service.server.stop(None) + @unittest.skipIf( sys.version_info[0] < 3 or sys.version_info[1] < 6, diff --git a/src/python/grpcio_tests/tests/qps/benchmark_client.py b/src/python/grpcio_tests/tests/qps/benchmark_client.py index e5aafc4142f..9310f8a8c61 100644 --- a/src/python/grpcio_tests/tests/qps/benchmark_client.py +++ b/src/python/grpcio_tests/tests/qps/benchmark_client.py @@ -32,13 +32,16 @@ _TIMEOUT = 60 * 60 * 24 class GenericStub(object): def __init__(self, channel): self.UnaryCall = channel.unary_unary( - "/grpc.testing.BenchmarkService/UnaryCall" + "/grpc.testing.BenchmarkService/UnaryCall", + _registered_method=True, ) self.StreamingFromServer = channel.unary_stream( - "/grpc.testing.BenchmarkService/StreamingFromServer" + "/grpc.testing.BenchmarkService/StreamingFromServer", + _registered_method=True, ) self.StreamingCall = channel.stream_stream( - "/grpc.testing.BenchmarkService/StreamingCall" + "/grpc.testing.BenchmarkService/StreamingCall", + _registered_method=True, ) diff --git a/src/python/grpcio_tests/tests/status/_grpc_status_test.py b/src/python/grpcio_tests/tests/status/_grpc_status_test.py index 2573e961f18..031bdbe4d53 100644 --- a/src/python/grpcio_tests/tests/status/_grpc_status_test.py +++ b/src/python/grpcio_tests/tests/status/_grpc_status_test.py @@ -138,7 +138,10 @@ class StatusTest(unittest.TestCase): self._channel.close() def test_status_ok(self): - _, call = self._channel.unary_unary(_STATUS_OK).with_call(_REQUEST) + _, call = self._channel.unary_unary( + _STATUS_OK, + _registered_method=True, + ).with_call(_REQUEST) # Succeed RPC doesn't have status status = rpc_status.from_call(call) @@ -146,7 +149,10 @@ class StatusTest(unittest.TestCase): def test_status_not_ok(self): with self.assertRaises(grpc.RpcError) as exception_context: - self._channel.unary_unary(_STATUS_NOT_OK).with_call(_REQUEST) + self._channel.unary_unary( + _STATUS_NOT_OK, + _registered_method=True, + ).with_call(_REQUEST) rpc_error = exception_context.exception self.assertEqual(rpc_error.code(), grpc.StatusCode.INTERNAL) @@ -156,7 +162,10 @@ class StatusTest(unittest.TestCase): def test_error_details(self): with self.assertRaises(grpc.RpcError) as exception_context: - self._channel.unary_unary(_ERROR_DETAILS).with_call(_REQUEST) + self._channel.unary_unary( + _ERROR_DETAILS, + _registered_method=True, + ).with_call(_REQUEST) rpc_error = exception_context.exception status = rpc_status.from_call(rpc_error) @@ -173,7 +182,10 @@ class StatusTest(unittest.TestCase): def test_code_message_validation(self): with self.assertRaises(grpc.RpcError) as exception_context: - self._channel.unary_unary(_INCONSISTENT).with_call(_REQUEST) + self._channel.unary_unary( + _INCONSISTENT, + _registered_method=True, + ).with_call(_REQUEST) rpc_error = exception_context.exception self.assertEqual(rpc_error.code(), grpc.StatusCode.NOT_FOUND) @@ -182,7 +194,10 @@ class StatusTest(unittest.TestCase): def test_invalid_code(self): with self.assertRaises(grpc.RpcError) as exception_context: - self._channel.unary_unary(_INVALID_CODE).with_call(_REQUEST) + self._channel.unary_unary( + _INVALID_CODE, + _registered_method=True, + ).with_call(_REQUEST) rpc_error = exception_context.exception self.assertEqual(rpc_error.code(), grpc.StatusCode.UNKNOWN) # Invalid status code exception raised during coversion diff --git a/src/python/grpcio_tests/tests/unit/_abort_test.py b/src/python/grpcio_tests/tests/unit/_abort_test.py index 731bb741bec..46f48bd1cae 100644 --- a/src/python/grpcio_tests/tests/unit/_abort_test.py +++ b/src/python/grpcio_tests/tests/unit/_abort_test.py @@ -107,7 +107,10 @@ class AbortTest(unittest.TestCase): def test_abort(self): with self.assertRaises(grpc.RpcError) as exception_context: - self._channel.unary_unary(_ABORT)(_REQUEST) + self._channel.unary_unary( + _ABORT, + _registered_method=True, + )(_REQUEST) rpc_error = exception_context.exception self.assertEqual(rpc_error.code(), grpc.StatusCode.INTERNAL) @@ -124,7 +127,10 @@ class AbortTest(unittest.TestCase): # Servicer will abort() after creating a local ref to do_not_leak_me. with self.assertRaises(grpc.RpcError): - self._channel.unary_unary(_ABORT)(_REQUEST) + self._channel.unary_unary( + _ABORT, + _registered_method=True, + )(_REQUEST) # Server may still have a stack frame reference to the exception even # after client sees error, so ensure server has shutdown. @@ -134,7 +140,10 @@ class AbortTest(unittest.TestCase): def test_abort_with_status(self): with self.assertRaises(grpc.RpcError) as exception_context: - self._channel.unary_unary(_ABORT_WITH_STATUS)(_REQUEST) + self._channel.unary_unary( + _ABORT_WITH_STATUS, + _registered_method=True, + )(_REQUEST) rpc_error = exception_context.exception self.assertEqual(rpc_error.code(), grpc.StatusCode.INTERNAL) @@ -143,7 +152,10 @@ class AbortTest(unittest.TestCase): def test_invalid_code(self): with self.assertRaises(grpc.RpcError) as exception_context: - self._channel.unary_unary(_INVALID_CODE)(_REQUEST) + self._channel.unary_unary( + _INVALID_CODE, + _registered_method=True, + )(_REQUEST) rpc_error = exception_context.exception self.assertEqual(rpc_error.code(), grpc.StatusCode.UNKNOWN) diff --git a/src/python/grpcio_tests/tests/unit/_auth_context_test.py b/src/python/grpcio_tests/tests/unit/_auth_context_test.py index 039c908c3e5..0e5e2017fba 100644 --- a/src/python/grpcio_tests/tests/unit/_auth_context_test.py +++ b/src/python/grpcio_tests/tests/unit/_auth_context_test.py @@ -78,7 +78,10 @@ class AuthContextTest(unittest.TestCase): server.start() with grpc.insecure_channel("localhost:%d" % port) as channel: - response = channel.unary_unary(_UNARY_UNARY)(_REQUEST) + response = channel.unary_unary( + _UNARY_UNARY, + _registered_method=True, + )(_REQUEST) server.stop(None) auth_data = pickle.loads(response) @@ -115,7 +118,10 @@ class AuthContextTest(unittest.TestCase): channel_creds, options=_PROPERTY_OPTIONS, ) - response = channel.unary_unary(_UNARY_UNARY)(_REQUEST) + response = channel.unary_unary( + _UNARY_UNARY, + _registered_method=True, + )(_REQUEST) channel.close() server.stop(None) @@ -161,7 +167,10 @@ class AuthContextTest(unittest.TestCase): options=_PROPERTY_OPTIONS, ) - response = channel.unary_unary(_UNARY_UNARY)(_REQUEST) + response = channel.unary_unary( + _UNARY_UNARY, + _registered_method=True, + )(_REQUEST) channel.close() server.stop(None) @@ -180,7 +189,10 @@ class AuthContextTest(unittest.TestCase): channel = grpc.secure_channel( "localhost:{}".format(port), channel_creds, options=channel_options ) - response = channel.unary_unary(_UNARY_UNARY)(_REQUEST) + response = channel.unary_unary( + _UNARY_UNARY, + _registered_method=True, + )(_REQUEST) auth_data = pickle.loads(response) self.assertEqual( expect_ssl_session_reused, diff --git a/src/python/grpcio_tests/tests/unit/_channel_close_test.py b/src/python/grpcio_tests/tests/unit/_channel_close_test.py index 4e5f215af89..b2ba4e7c887 100644 --- a/src/python/grpcio_tests/tests/unit/_channel_close_test.py +++ b/src/python/grpcio_tests/tests/unit/_channel_close_test.py @@ -123,7 +123,10 @@ class ChannelCloseTest(unittest.TestCase): def test_close_immediately_after_call_invocation(self): channel = grpc.insecure_channel("localhost:{}".format(self._port)) - multi_callable = channel.stream_stream(_STREAM_URI) + multi_callable = channel.stream_stream( + _STREAM_URI, + _registered_method=True, + ) request_iterator = _Pipe(()) response_iterator = multi_callable(request_iterator) channel.close() @@ -133,7 +136,10 @@ class ChannelCloseTest(unittest.TestCase): def test_close_while_call_active(self): channel = grpc.insecure_channel("localhost:{}".format(self._port)) - multi_callable = channel.stream_stream(_STREAM_URI) + multi_callable = channel.stream_stream( + _STREAM_URI, + _registered_method=True, + ) request_iterator = _Pipe((b"abc",)) response_iterator = multi_callable(request_iterator) next(response_iterator) @@ -146,7 +152,10 @@ class ChannelCloseTest(unittest.TestCase): with grpc.insecure_channel( "localhost:{}".format(self._port) ) as channel: # pylint: disable=bad-continuation - multi_callable = channel.stream_stream(_STREAM_URI) + multi_callable = channel.stream_stream( + _STREAM_URI, + _registered_method=True, + ) request_iterator = _Pipe((b"abc",)) response_iterator = multi_callable(request_iterator) next(response_iterator) @@ -158,7 +167,10 @@ class ChannelCloseTest(unittest.TestCase): with grpc.insecure_channel( "localhost:{}".format(self._port) ) as channel: # pylint: disable=bad-continuation - multi_callable = channel.stream_stream(_STREAM_URI) + multi_callable = channel.stream_stream( + _STREAM_URI, + _registered_method=True, + ) request_iterators = tuple( _Pipe((b"abc",)) for _ in range(test_constants.THREAD_CONCURRENCY) @@ -176,7 +188,10 @@ class ChannelCloseTest(unittest.TestCase): def test_many_concurrent_closes(self): channel = grpc.insecure_channel("localhost:{}".format(self._port)) - multi_callable = channel.stream_stream(_STREAM_URI) + multi_callable = channel.stream_stream( + _STREAM_URI, + _registered_method=True, + ) request_iterator = _Pipe((b"abc",)) response_iterator = multi_callable(request_iterator) next(response_iterator) @@ -203,10 +218,16 @@ class ChannelCloseTest(unittest.TestCase): with grpc.insecure_channel( "localhost:{}".format(self._port) ) as channel: - stream_multi_callable = channel.stream_stream(_STREAM_URI) + stream_multi_callable = channel.stream_stream( + _STREAM_URI, + _registered_method=True, + ) endless_iterator = itertools.repeat(b"abc") stream_response_iterator = stream_multi_callable(endless_iterator) - future = channel.unary_unary(_UNARY_URI).future(b"abc") + future = channel.unary_unary( + _UNARY_URI, + _registered_method=True, + ).future(b"abc") def on_done_callback(future): raise Exception("This should not cause a deadlock.") diff --git a/src/python/grpcio_tests/tests/unit/_compression_test.py b/src/python/grpcio_tests/tests/unit/_compression_test.py index be2a528ea9b..9fdeca030d8 100644 --- a/src/python/grpcio_tests/tests/unit/_compression_test.py +++ b/src/python/grpcio_tests/tests/unit/_compression_test.py @@ -237,7 +237,10 @@ def _get_compression_ratios( def _unary_unary_client(channel, multicallable_kwargs, message): - multi_callable = channel.unary_unary(_UNARY_UNARY) + multi_callable = channel.unary_unary( + _UNARY_UNARY, + _registered_method=True, + ) response = multi_callable(message, **multicallable_kwargs) if response != message: raise RuntimeError( @@ -246,7 +249,10 @@ def _unary_unary_client(channel, multicallable_kwargs, message): def _unary_stream_client(channel, multicallable_kwargs, message): - multi_callable = channel.unary_stream(_UNARY_STREAM) + multi_callable = channel.unary_stream( + _UNARY_STREAM, + _registered_method=True, + ) response_iterator = multi_callable(message, **multicallable_kwargs) for response in response_iterator: if response != message: @@ -256,7 +262,10 @@ def _unary_stream_client(channel, multicallable_kwargs, message): def _stream_unary_client(channel, multicallable_kwargs, message): - multi_callable = channel.stream_unary(_STREAM_UNARY) + multi_callable = channel.stream_unary( + _STREAM_UNARY, + _registered_method=True, + ) requests = (_REQUEST for _ in range(_STREAM_LENGTH)) response = multi_callable(requests, **multicallable_kwargs) if response != message: @@ -266,7 +275,10 @@ def _stream_unary_client(channel, multicallable_kwargs, message): def _stream_stream_client(channel, multicallable_kwargs, message): - multi_callable = channel.stream_stream(_STREAM_STREAM) + multi_callable = channel.stream_stream( + _STREAM_STREAM, + _registered_method=True, + ) request_prefix = str(0).encode("ascii") * 100 requests = ( request_prefix + str(i).encode("ascii") for i in range(_STREAM_LENGTH) diff --git a/src/python/grpcio_tests/tests/unit/_contextvars_propagation_test.py b/src/python/grpcio_tests/tests/unit/_contextvars_propagation_test.py index 6f3b601ceb2..5a23d5dc69e 100644 --- a/src/python/grpcio_tests/tests/unit/_contextvars_propagation_test.py +++ b/src/python/grpcio_tests/tests/unit/_contextvars_propagation_test.py @@ -116,7 +116,10 @@ class ContextVarsPropagationTest(unittest.TestCase): local_credentials, call_credentials ) with grpc.secure_channel(target, composite_credentials) as channel: - stub = channel.unary_unary(_UNARY_UNARY) + stub = channel.unary_unary( + _UNARY_UNARY, + _registered_method=True, + ) response = stub(_REQUEST, wait_for_ready=True) self.assertEqual(_REQUEST, response) @@ -142,7 +145,10 @@ class ContextVarsPropagationTest(unittest.TestCase): with grpc.secure_channel( target, composite_credentials ) as channel: - stub = channel.unary_unary(_UNARY_UNARY) + stub = channel.unary_unary( + _UNARY_UNARY, + _registered_method=True, + ) wait_group.done() wait_group.wait() for i in range(_RPC_COUNT): diff --git a/src/python/grpcio_tests/tests/unit/_dns_resolver_test.py b/src/python/grpcio_tests/tests/unit/_dns_resolver_test.py index 62a95a02135..bcd7e6da849 100644 --- a/src/python/grpcio_tests/tests/unit/_dns_resolver_test.py +++ b/src/python/grpcio_tests/tests/unit/_dns_resolver_test.py @@ -55,7 +55,10 @@ class DNSResolverTest(unittest.TestCase): "loopback46.unittest.grpc.io:%d" % self._port ) as channel: self.assertEqual( - channel.unary_unary(_METHOD)( + channel.unary_unary( + _METHOD, + _registered_method=True, + )( _REQUEST, timeout=10, ), diff --git a/src/python/grpcio_tests/tests/unit/_empty_message_test.py b/src/python/grpcio_tests/tests/unit/_empty_message_test.py index e2dc1594202..a303aa8b3e9 100644 --- a/src/python/grpcio_tests/tests/unit/_empty_message_test.py +++ b/src/python/grpcio_tests/tests/unit/_empty_message_test.py @@ -96,25 +96,33 @@ class EmptyMessageTest(unittest.TestCase): self._channel.close() def testUnaryUnary(self): - response = self._channel.unary_unary(_UNARY_UNARY)(_REQUEST) + response = self._channel.unary_unary( + _UNARY_UNARY, + _registered_method=True, + )(_REQUEST) self.assertEqual(_RESPONSE, response) def testUnaryStream(self): - response_iterator = self._channel.unary_stream(_UNARY_STREAM)(_REQUEST) + response_iterator = self._channel.unary_stream( + _UNARY_STREAM, + _registered_method=True, + )(_REQUEST) self.assertSequenceEqual( [_RESPONSE] * test_constants.STREAM_LENGTH, list(response_iterator) ) def testStreamUnary(self): - response = self._channel.stream_unary(_STREAM_UNARY)( - iter([_REQUEST] * test_constants.STREAM_LENGTH) - ) + response = self._channel.stream_unary( + _STREAM_UNARY, + _registered_method=True, + )(iter([_REQUEST] * test_constants.STREAM_LENGTH)) self.assertEqual(_RESPONSE, response) def testStreamStream(self): - response_iterator = self._channel.stream_stream(_STREAM_STREAM)( - iter([_REQUEST] * test_constants.STREAM_LENGTH) - ) + response_iterator = self._channel.stream_stream( + _STREAM_STREAM, + _registered_method=True, + )(iter([_REQUEST] * test_constants.STREAM_LENGTH)) self.assertSequenceEqual( [_RESPONSE] * test_constants.STREAM_LENGTH, list(response_iterator) ) diff --git a/src/python/grpcio_tests/tests/unit/_error_message_encoding_test.py b/src/python/grpcio_tests/tests/unit/_error_message_encoding_test.py index 4f07477fac9..334b7ed5a3a 100644 --- a/src/python/grpcio_tests/tests/unit/_error_message_encoding_test.py +++ b/src/python/grpcio_tests/tests/unit/_error_message_encoding_test.py @@ -73,7 +73,10 @@ class ErrorMessageEncodingTest(unittest.TestCase): def testMessageEncoding(self): for message in _UNICODE_ERROR_MESSAGES: - multi_callable = self._channel.unary_unary(_UNARY_UNARY) + multi_callable = self._channel.unary_unary( + _UNARY_UNARY, + _registered_method=True, + ) with self.assertRaises(grpc.RpcError) as cm: multi_callable(message.encode("utf-8")) diff --git a/src/python/grpcio_tests/tests/unit/_exit_scenarios.py b/src/python/grpcio_tests/tests/unit/_exit_scenarios.py index c1f9816df08..1b7e2e5ac84 100644 --- a/src/python/grpcio_tests/tests/unit/_exit_scenarios.py +++ b/src/python/grpcio_tests/tests/unit/_exit_scenarios.py @@ -210,14 +210,20 @@ if __name__ == "__main__": method = TEST_TO_METHOD[args.scenario] if args.scenario == IN_FLIGHT_UNARY_UNARY_CALL: - multi_callable = channel.unary_unary(method) + multi_callable = channel.unary_unary( + method, + _registered_method=True, + ) future = multi_callable.future(REQUEST) result, call = multi_callable.with_call(REQUEST) elif ( args.scenario == IN_FLIGHT_UNARY_STREAM_CALL or args.scenario == IN_FLIGHT_PARTIAL_UNARY_STREAM_CALL ): - multi_callable = channel.unary_stream(method) + multi_callable = channel.unary_stream( + method, + _registered_method=True, + ) response_iterator = multi_callable(REQUEST) for response in response_iterator: pass @@ -225,7 +231,10 @@ if __name__ == "__main__": args.scenario == IN_FLIGHT_STREAM_UNARY_CALL or args.scenario == IN_FLIGHT_PARTIAL_STREAM_UNARY_CALL ): - multi_callable = channel.stream_unary(method) + multi_callable = channel.stream_unary( + method, + _registered_method=True, + ) future = multi_callable.future(infinite_request_iterator()) result, call = multi_callable.with_call( iter([REQUEST] * test_constants.STREAM_LENGTH) @@ -234,7 +243,10 @@ if __name__ == "__main__": args.scenario == IN_FLIGHT_STREAM_STREAM_CALL or args.scenario == IN_FLIGHT_PARTIAL_STREAM_STREAM_CALL ): - multi_callable = channel.stream_stream(method) + multi_callable = channel.stream_stream( + method, + _registered_method=True, + ) response_iterator = multi_callable(infinite_request_iterator()) for response in response_iterator: pass diff --git a/src/python/grpcio_tests/tests/unit/_interceptor_test.py b/src/python/grpcio_tests/tests/unit/_interceptor_test.py index 9bbff1f6bee..72e299b5887 100644 --- a/src/python/grpcio_tests/tests/unit/_interceptor_test.py +++ b/src/python/grpcio_tests/tests/unit/_interceptor_test.py @@ -231,7 +231,7 @@ class _GenericHandler(grpc.GenericRpcHandler): def _unary_unary_multi_callable(channel): - return channel.unary_unary(_UNARY_UNARY) + return channel.unary_unary(_UNARY_UNARY, _registered_method=True) def _unary_stream_multi_callable(channel): @@ -239,6 +239,7 @@ def _unary_stream_multi_callable(channel): _UNARY_STREAM, request_serializer=_SERIALIZE_REQUEST, response_deserializer=_DESERIALIZE_RESPONSE, + _registered_method=True, ) @@ -247,11 +248,12 @@ def _stream_unary_multi_callable(channel): _STREAM_UNARY, request_serializer=_SERIALIZE_REQUEST, response_deserializer=_DESERIALIZE_RESPONSE, + _registered_method=True, ) def _stream_stream_multi_callable(channel): - return channel.stream_stream(_STREAM_STREAM) + return channel.stream_stream(_STREAM_STREAM, _registered_method=True) class _ClientCallDetails( @@ -562,7 +564,7 @@ class InterceptorTest(unittest.TestCase): self._record[:] = [] multi_callable = _unary_unary_multi_callable(channel) - multi_callable.with_call( + response, call = multi_callable.with_call( request, metadata=( ( diff --git a/src/python/grpcio_tests/tests/unit/_invalid_metadata_test.py b/src/python/grpcio_tests/tests/unit/_invalid_metadata_test.py index a19966131c5..58d1e589fde 100644 --- a/src/python/grpcio_tests/tests/unit/_invalid_metadata_test.py +++ b/src/python/grpcio_tests/tests/unit/_invalid_metadata_test.py @@ -32,7 +32,7 @@ _STREAM_STREAM = "/test/StreamStream" def _unary_unary_multi_callable(channel): - return channel.unary_unary(_UNARY_UNARY) + return channel.unary_unary(_UNARY_UNARY, _registered_method=True) def _unary_stream_multi_callable(channel): @@ -40,6 +40,7 @@ def _unary_stream_multi_callable(channel): _UNARY_STREAM, request_serializer=_SERIALIZE_REQUEST, response_deserializer=_DESERIALIZE_RESPONSE, + _registered_method=True, ) @@ -48,11 +49,15 @@ def _stream_unary_multi_callable(channel): _STREAM_UNARY, request_serializer=_SERIALIZE_REQUEST, response_deserializer=_DESERIALIZE_RESPONSE, + _registered_method=True, ) def _stream_stream_multi_callable(channel): - return channel.stream_stream(_STREAM_STREAM) + return channel.stream_stream( + _STREAM_STREAM, + _registered_method=True, + ) class InvalidMetadataTest(unittest.TestCase): diff --git a/src/python/grpcio_tests/tests/unit/_invocation_defects_test.py b/src/python/grpcio_tests/tests/unit/_invocation_defects_test.py index b22ab016593..cb903e7b812 100644 --- a/src/python/grpcio_tests/tests/unit/_invocation_defects_test.py +++ b/src/python/grpcio_tests/tests/unit/_invocation_defects_test.py @@ -219,7 +219,10 @@ class FailAfterFewIterationsCounter(object): def _unary_unary_multi_callable(channel): - return channel.unary_unary(_UNARY_UNARY) + return channel.unary_unary( + _UNARY_UNARY, + _registered_method=True, + ) def _unary_stream_multi_callable(channel): @@ -227,6 +230,7 @@ def _unary_stream_multi_callable(channel): _UNARY_STREAM, request_serializer=_SERIALIZE_REQUEST, response_deserializer=_DESERIALIZE_RESPONSE, + _registered_method=True, ) @@ -235,19 +239,29 @@ def _stream_unary_multi_callable(channel): _STREAM_UNARY, request_serializer=_SERIALIZE_REQUEST, response_deserializer=_DESERIALIZE_RESPONSE, + _registered_method=True, ) def _stream_stream_multi_callable(channel): - return channel.stream_stream(_STREAM_STREAM) + return channel.stream_stream( + _STREAM_STREAM, + _registered_method=True, + ) def _defective_handler_multi_callable(channel): - return channel.unary_unary(_DEFECTIVE_GENERIC_RPC_HANDLER) + return channel.unary_unary( + _DEFECTIVE_GENERIC_RPC_HANDLER, + _registered_method=True, + ) def _defective_nested_exception_handler_multi_callable(channel): - return channel.unary_unary(_UNARY_UNARY_NESTED_EXCEPTION) + return channel.unary_unary( + _UNARY_UNARY_NESTED_EXCEPTION, + _registered_method=True, + ) class InvocationDefectsTest(unittest.TestCase): diff --git a/src/python/grpcio_tests/tests/unit/_local_credentials_test.py b/src/python/grpcio_tests/tests/unit/_local_credentials_test.py index 165f6ca16eb..9c5b425eaf7 100644 --- a/src/python/grpcio_tests/tests/unit/_local_credentials_test.py +++ b/src/python/grpcio_tests/tests/unit/_local_credentials_test.py @@ -53,9 +53,10 @@ class LocalCredentialsTest(unittest.TestCase): ) as channel: self.assertEqual( b"abc", - channel.unary_unary("/test/method")( - b"abc", wait_for_ready=True - ), + channel.unary_unary( + "/test/method", + _registered_method=True, + )(b"abc", wait_for_ready=True), ) server.stop(None) @@ -77,9 +78,10 @@ class LocalCredentialsTest(unittest.TestCase): with grpc.secure_channel(server_addr, channel_creds) as channel: self.assertEqual( b"abc", - channel.unary_unary("/test/method")( - b"abc", wait_for_ready=True - ), + channel.unary_unary( + "/test/method", + _registered_method=True, + )(b"abc", wait_for_ready=True), ) server.stop(None) diff --git a/src/python/grpcio_tests/tests/unit/_metadata_code_details_test.py b/src/python/grpcio_tests/tests/unit/_metadata_code_details_test.py index 3c530058dc3..320deb7e5f6 100644 --- a/src/python/grpcio_tests/tests/unit/_metadata_code_details_test.py +++ b/src/python/grpcio_tests/tests/unit/_metadata_code_details_test.py @@ -207,45 +207,53 @@ class MetadataCodeDetailsTest(unittest.TestCase): self._server.start() self._channel = grpc.insecure_channel("localhost:{}".format(port)) + unary_unary_method_name = "/".join( + ( + "", + _SERVICE, + _UNARY_UNARY, + ) + ) self._unary_unary = self._channel.unary_unary( - "/".join( - ( - "", - _SERVICE, - _UNARY_UNARY, - ) - ), + unary_unary_method_name, request_serializer=_REQUEST_SERIALIZER, response_deserializer=_RESPONSE_DESERIALIZER, + _registered_method=True, + ) + unary_stream_method_name = "/".join( + ( + "", + _SERVICE, + _UNARY_STREAM, + ) ) self._unary_stream = self._channel.unary_stream( - "/".join( - ( - "", - _SERVICE, - _UNARY_STREAM, - ) - ), + unary_stream_method_name, + _registered_method=True, + ) + stream_unary_method_name = "/".join( + ( + "", + _SERVICE, + _STREAM_UNARY, + ) ) self._stream_unary = self._channel.stream_unary( - "/".join( - ( - "", - _SERVICE, - _STREAM_UNARY, - ) - ), + stream_unary_method_name, + _registered_method=True, + ) + stream_stream_method_name = "/".join( + ( + "", + _SERVICE, + _STREAM_STREAM, + ) ) self._stream_stream = self._channel.stream_stream( - "/".join( - ( - "", - _SERVICE, - _STREAM_STREAM, - ) - ), + stream_stream_method_name, request_serializer=_REQUEST_SERIALIZER, response_deserializer=_RESPONSE_DESERIALIZER, + _registered_method=True, ) def tearDown(self): @@ -828,16 +836,18 @@ class InspectContextTest(unittest.TestCase): self._server.start() self._channel = grpc.insecure_channel("localhost:{}".format(port)) + unary_unary_method_name = "/".join( + ( + "", + _SERVICE, + _UNARY_UNARY, + ) + ) self._unary_unary = self._channel.unary_unary( - "/".join( - ( - "", - _SERVICE, - _UNARY_UNARY, - ) - ), + unary_unary_method_name, request_serializer=_REQUEST_SERIALIZER, response_deserializer=_RESPONSE_DESERIALIZER, + _registered_method=True, ) def tearDown(self): diff --git a/src/python/grpcio_tests/tests/unit/_metadata_flags_test.py b/src/python/grpcio_tests/tests/unit/_metadata_flags_test.py index a67a496860f..2cd9ad9bd89 100644 --- a/src/python/grpcio_tests/tests/unit/_metadata_flags_test.py +++ b/src/python/grpcio_tests/tests/unit/_metadata_flags_test.py @@ -110,7 +110,10 @@ def create_phony_channel(): def perform_unary_unary_call(channel, wait_for_ready=None): - channel.unary_unary(_UNARY_UNARY).__call__( + channel.unary_unary( + _UNARY_UNARY, + _registered_method=True, + ).__call__( _REQUEST, timeout=test_constants.LONG_TIMEOUT, wait_for_ready=wait_for_ready, @@ -118,7 +121,10 @@ def perform_unary_unary_call(channel, wait_for_ready=None): def perform_unary_unary_with_call(channel, wait_for_ready=None): - channel.unary_unary(_UNARY_UNARY).with_call( + channel.unary_unary( + _UNARY_UNARY, + _registered_method=True, + ).with_call( _REQUEST, timeout=test_constants.LONG_TIMEOUT, wait_for_ready=wait_for_ready, @@ -126,7 +132,10 @@ def perform_unary_unary_with_call(channel, wait_for_ready=None): def perform_unary_unary_future(channel, wait_for_ready=None): - channel.unary_unary(_UNARY_UNARY).future( + channel.unary_unary( + _UNARY_UNARY, + _registered_method=True, + ).future( _REQUEST, timeout=test_constants.LONG_TIMEOUT, wait_for_ready=wait_for_ready, @@ -134,7 +143,10 @@ def perform_unary_unary_future(channel, wait_for_ready=None): def perform_unary_stream_call(channel, wait_for_ready=None): - response_iterator = channel.unary_stream(_UNARY_STREAM).__call__( + response_iterator = channel.unary_stream( + _UNARY_STREAM, + _registered_method=True, + ).__call__( _REQUEST, timeout=test_constants.LONG_TIMEOUT, wait_for_ready=wait_for_ready, @@ -144,7 +156,10 @@ def perform_unary_stream_call(channel, wait_for_ready=None): def perform_stream_unary_call(channel, wait_for_ready=None): - channel.stream_unary(_STREAM_UNARY).__call__( + channel.stream_unary( + _STREAM_UNARY, + _registered_method=True, + ).__call__( iter([_REQUEST] * test_constants.STREAM_LENGTH), timeout=test_constants.LONG_TIMEOUT, wait_for_ready=wait_for_ready, @@ -152,7 +167,10 @@ def perform_stream_unary_call(channel, wait_for_ready=None): def perform_stream_unary_with_call(channel, wait_for_ready=None): - channel.stream_unary(_STREAM_UNARY).with_call( + channel.stream_unary( + _STREAM_UNARY, + _registered_method=True, + ).with_call( iter([_REQUEST] * test_constants.STREAM_LENGTH), timeout=test_constants.LONG_TIMEOUT, wait_for_ready=wait_for_ready, @@ -160,7 +178,10 @@ def perform_stream_unary_with_call(channel, wait_for_ready=None): def perform_stream_unary_future(channel, wait_for_ready=None): - channel.stream_unary(_STREAM_UNARY).future( + channel.stream_unary( + _STREAM_UNARY, + _registered_method=True, + ).future( iter([_REQUEST] * test_constants.STREAM_LENGTH), timeout=test_constants.LONG_TIMEOUT, wait_for_ready=wait_for_ready, @@ -168,7 +189,9 @@ def perform_stream_unary_future(channel, wait_for_ready=None): def perform_stream_stream_call(channel, wait_for_ready=None): - response_iterator = channel.stream_stream(_STREAM_STREAM).__call__( + response_iterator = channel.stream_stream( + _STREAM_STREAM, _registered_method=True + ).__call__( iter([_REQUEST] * test_constants.STREAM_LENGTH), timeout=test_constants.LONG_TIMEOUT, wait_for_ready=wait_for_ready, diff --git a/src/python/grpcio_tests/tests/unit/_metadata_test.py b/src/python/grpcio_tests/tests/unit/_metadata_test.py index 7110177fa18..b9b7502972c 100644 --- a/src/python/grpcio_tests/tests/unit/_metadata_test.py +++ b/src/python/grpcio_tests/tests/unit/_metadata_test.py @@ -195,7 +195,9 @@ class MetadataTest(unittest.TestCase): self._channel.close() def testUnaryUnary(self): - multi_callable = self._channel.unary_unary(_UNARY_UNARY) + multi_callable = self._channel.unary_unary( + _UNARY_UNARY, _registered_method=True + ) unused_response, call = multi_callable.with_call( _REQUEST, metadata=_INVOCATION_METADATA ) @@ -211,7 +213,9 @@ class MetadataTest(unittest.TestCase): ) def testUnaryStream(self): - multi_callable = self._channel.unary_stream(_UNARY_STREAM) + multi_callable = self._channel.unary_stream( + _UNARY_STREAM, _registered_method=True + ) call = multi_callable(_REQUEST, metadata=_INVOCATION_METADATA) self.assertTrue( test_common.metadata_transmitted( @@ -227,7 +231,9 @@ class MetadataTest(unittest.TestCase): ) def testStreamUnary(self): - multi_callable = self._channel.stream_unary(_STREAM_UNARY) + multi_callable = self._channel.stream_unary( + _STREAM_UNARY, _registered_method=True + ) unused_response, call = multi_callable.with_call( iter([_REQUEST] * test_constants.STREAM_LENGTH), metadata=_INVOCATION_METADATA, @@ -244,7 +250,9 @@ class MetadataTest(unittest.TestCase): ) def testStreamStream(self): - multi_callable = self._channel.stream_stream(_STREAM_STREAM) + multi_callable = self._channel.stream_stream( + _STREAM_STREAM, _registered_method=True + ) call = multi_callable( iter([_REQUEST] * test_constants.STREAM_LENGTH), metadata=_INVOCATION_METADATA, diff --git a/src/python/grpcio_tests/tests/unit/_reconnect_test.py b/src/python/grpcio_tests/tests/unit/_reconnect_test.py index d412533251c..8d4fadaa3da 100644 --- a/src/python/grpcio_tests/tests/unit/_reconnect_test.py +++ b/src/python/grpcio_tests/tests/unit/_reconnect_test.py @@ -52,7 +52,10 @@ class ReconnectTest(unittest.TestCase): server.add_insecure_port(addr) server.start() channel = grpc.insecure_channel(addr) - multi_callable = channel.unary_unary(_UNARY_UNARY) + multi_callable = channel.unary_unary( + _UNARY_UNARY, + _registered_method=True, + ) self.assertEqual(_RESPONSE, multi_callable(_REQUEST)) server.stop(None) # By default, the channel connectivity is checked every 5s diff --git a/src/python/grpcio_tests/tests/unit/_resource_exhausted_test.py b/src/python/grpcio_tests/tests/unit/_resource_exhausted_test.py index 3fc04f06a18..93e72c84f07 100644 --- a/src/python/grpcio_tests/tests/unit/_resource_exhausted_test.py +++ b/src/python/grpcio_tests/tests/unit/_resource_exhausted_test.py @@ -149,7 +149,10 @@ class ResourceExhaustedTest(unittest.TestCase): self._channel.close() def testUnaryUnary(self): - multi_callable = self._channel.unary_unary(_UNARY_UNARY) + multi_callable = self._channel.unary_unary( + _UNARY_UNARY, + _registered_method=True, + ) futures = [] for _ in range(test_constants.THREAD_CONCURRENCY): futures.append(multi_callable.future(_REQUEST)) @@ -178,7 +181,10 @@ class ResourceExhaustedTest(unittest.TestCase): self.assertEqual(_RESPONSE, multi_callable(_REQUEST)) def testUnaryStream(self): - multi_callable = self._channel.unary_stream(_UNARY_STREAM) + multi_callable = self._channel.unary_stream( + _UNARY_STREAM, + _registered_method=True, + ) calls = [] for _ in range(test_constants.THREAD_CONCURRENCY): calls.append(multi_callable(_REQUEST)) @@ -205,7 +211,10 @@ class ResourceExhaustedTest(unittest.TestCase): self.assertEqual(_RESPONSE, response) def testStreamUnary(self): - multi_callable = self._channel.stream_unary(_STREAM_UNARY) + multi_callable = self._channel.stream_unary( + _STREAM_UNARY, + _registered_method=True, + ) futures = [] request = iter([_REQUEST] * test_constants.STREAM_LENGTH) for _ in range(test_constants.THREAD_CONCURRENCY): @@ -236,7 +245,10 @@ class ResourceExhaustedTest(unittest.TestCase): self.assertEqual(_RESPONSE, multi_callable(request)) def testStreamStream(self): - multi_callable = self._channel.stream_stream(_STREAM_STREAM) + multi_callable = self._channel.stream_stream( + _STREAM_STREAM, + _registered_method=True, + ) calls = [] request = iter([_REQUEST] * test_constants.STREAM_LENGTH) for _ in range(test_constants.THREAD_CONCURRENCY): diff --git a/src/python/grpcio_tests/tests/unit/_rpc_test_helpers.py b/src/python/grpcio_tests/tests/unit/_rpc_test_helpers.py index 1027be1c677..d85eb30c8e7 100644 --- a/src/python/grpcio_tests/tests/unit/_rpc_test_helpers.py +++ b/src/python/grpcio_tests/tests/unit/_rpc_test_helpers.py @@ -277,7 +277,10 @@ class _GenericHandler(grpc.GenericRpcHandler): def unary_unary_multi_callable(channel): - return channel.unary_unary(_UNARY_UNARY) + return channel.unary_unary( + _UNARY_UNARY, + _registered_method=True, + ) def unary_stream_multi_callable(channel): @@ -285,6 +288,7 @@ def unary_stream_multi_callable(channel): _UNARY_STREAM, request_serializer=_SERIALIZE_REQUEST, response_deserializer=_DESERIALIZE_RESPONSE, + _registered_method=True, ) @@ -293,6 +297,7 @@ def unary_stream_non_blocking_multi_callable(channel): _UNARY_STREAM_NON_BLOCKING, request_serializer=_SERIALIZE_REQUEST, response_deserializer=_DESERIALIZE_RESPONSE, + _registered_method=True, ) @@ -301,15 +306,22 @@ def stream_unary_multi_callable(channel): _STREAM_UNARY, request_serializer=_SERIALIZE_REQUEST, response_deserializer=_DESERIALIZE_RESPONSE, + _registered_method=True, ) def stream_stream_multi_callable(channel): - return channel.stream_stream(_STREAM_STREAM) + return channel.stream_stream( + _STREAM_STREAM, + _registered_method=True, + ) def stream_stream_non_blocking_multi_callable(channel): - return channel.stream_stream(_STREAM_STREAM_NON_BLOCKING) + return channel.stream_stream( + _STREAM_STREAM_NON_BLOCKING, + _registered_method=True, + ) class BaseRPCTest(object): diff --git a/src/python/grpcio_tests/tests/unit/_server_shutdown_scenarios.py b/src/python/grpcio_tests/tests/unit/_server_shutdown_scenarios.py index 9190f108f7b..34d51fd72ec 100644 --- a/src/python/grpcio_tests/tests/unit/_server_shutdown_scenarios.py +++ b/src/python/grpcio_tests/tests/unit/_server_shutdown_scenarios.py @@ -81,7 +81,10 @@ def run_test(args): thread.start() port = port_queue.get() channel = grpc.insecure_channel("localhost:%d" % port) - multi_callable = channel.unary_unary(FORK_EXIT) + multi_callable = channel.unary_unary( + FORK_EXIT, + _registered_method=True, + ) result, call = multi_callable.with_call(REQUEST, wait_for_ready=True) os.wait() else: diff --git a/src/python/grpcio_tests/tests/unit/_session_cache_test.py b/src/python/grpcio_tests/tests/unit/_session_cache_test.py index acf671d1ca9..e4bc64f7d11 100644 --- a/src/python/grpcio_tests/tests/unit/_session_cache_test.py +++ b/src/python/grpcio_tests/tests/unit/_session_cache_test.py @@ -77,7 +77,10 @@ class SSLSessionCacheTest(unittest.TestCase): channel = grpc.secure_channel( "localhost:{}".format(port), channel_creds, options=channel_options ) - response = channel.unary_unary(_UNARY_UNARY)(_REQUEST) + response = channel.unary_unary( + _UNARY_UNARY, + _registered_method=True, + )(_REQUEST) auth_data = pickle.loads(response) self.assertEqual( expect_ssl_session_reused, diff --git a/src/python/grpcio_tests/tests/unit/_signal_client.py b/src/python/grpcio_tests/tests/unit/_signal_client.py index 56563c20075..34c3da0c933 100644 --- a/src/python/grpcio_tests/tests/unit/_signal_client.py +++ b/src/python/grpcio_tests/tests/unit/_signal_client.py @@ -53,7 +53,10 @@ def main_unary(server_target): """Initiate a unary RPC to be interrupted by a SIGINT.""" global per_process_rpc_future # pylint: disable=global-statement with grpc.insecure_channel(server_target) as channel: - multicallable = channel.unary_unary(UNARY_UNARY) + multicallable = channel.unary_unary( + UNARY_UNARY, + _registered_method=True, + ) signal.signal(signal.SIGINT, handle_sigint) per_process_rpc_future = multicallable.future( _MESSAGE, wait_for_ready=True @@ -67,9 +70,10 @@ def main_streaming(server_target): global per_process_rpc_future # pylint: disable=global-statement with grpc.insecure_channel(server_target) as channel: signal.signal(signal.SIGINT, handle_sigint) - per_process_rpc_future = channel.unary_stream(UNARY_STREAM)( - _MESSAGE, wait_for_ready=True - ) + per_process_rpc_future = channel.unary_stream( + UNARY_STREAM, + _registered_method=True, + )(_MESSAGE, wait_for_ready=True) for result in per_process_rpc_future: pass assert False, _ASSERTION_MESSAGE @@ -79,7 +83,10 @@ def main_unary_with_exception(server_target): """Initiate a unary RPC with a signal handler that will raise.""" channel = grpc.insecure_channel(server_target) try: - channel.unary_unary(UNARY_UNARY)(_MESSAGE, wait_for_ready=True) + channel.unary_unary( + UNARY_UNARY, + _registered_method=True, + )(_MESSAGE, wait_for_ready=True) except KeyboardInterrupt: sys.stderr.write("Running signal handler.\n") sys.stderr.flush() @@ -92,9 +99,10 @@ def main_streaming_with_exception(server_target): """Initiate a streaming RPC with a signal handler that will raise.""" channel = grpc.insecure_channel(server_target) try: - for _ in channel.unary_stream(UNARY_STREAM)( - _MESSAGE, wait_for_ready=True - ): + for _ in channel.unary_stream( + UNARY_STREAM, + _registered_method=True, + )(_MESSAGE, wait_for_ready=True): pass except KeyboardInterrupt: sys.stderr.write("Running signal handler.\n") diff --git a/src/python/grpcio_tests/tests/unit/_xds_credentials_test.py b/src/python/grpcio_tests/tests/unit/_xds_credentials_test.py index 977d564888d..6d8b2b6a041 100644 --- a/src/python/grpcio_tests/tests/unit/_xds_credentials_test.py +++ b/src/python/grpcio_tests/tests/unit/_xds_credentials_test.py @@ -71,9 +71,10 @@ class XdsCredentialsTest(unittest.TestCase): server_address, channel_creds, options=override_options ) as channel: request = b"abc" - response = channel.unary_unary("/test/method")( - request, wait_for_ready=True - ) + response = channel.unary_unary( + "/test/method", + _registered_method=True, + )(request, wait_for_ready=True) self.assertEqual(response, request) def test_xds_creds_fallback_insecure(self): @@ -89,9 +90,10 @@ class XdsCredentialsTest(unittest.TestCase): channel_creds = grpc.xds_channel_credentials(channel_fallback_creds) with grpc.secure_channel(server_address, channel_creds) as channel: request = b"abc" - response = channel.unary_unary("/test/method")( - request, wait_for_ready=True - ) + response = channel.unary_unary( + "/test/method", + _registered_method=True, + )(request, wait_for_ready=True) self.assertEqual(response, request) def test_start_xds_server(self): diff --git a/src/python/grpcio_tests/tests_gevent/unit/close_channel_test.py b/src/python/grpcio_tests/tests_gevent/unit/close_channel_test.py index 47fdb2c22e7..f09c47734e0 100644 --- a/src/python/grpcio_tests/tests_gevent/unit/close_channel_test.py +++ b/src/python/grpcio_tests/tests_gevent/unit/close_channel_test.py @@ -65,6 +65,7 @@ class CloseChannelTest(unittest.TestCase): _UNARY_CALL_METHOD_WITH_SLEEP, request_serializer=messages_pb2.SimpleRequest.SerializeToString, response_deserializer=messages_pb2.SimpleResponse.FromString, + _registered_method=True, ) greenlet = group.spawn(self._run_client, UnaryCallWithSleep) # release loop so that greenlet can take control @@ -78,6 +79,7 @@ class CloseChannelTest(unittest.TestCase): _UNARY_CALL_METHOD_WITH_SLEEP, request_serializer=messages_pb2.SimpleRequest.SerializeToString, response_deserializer=messages_pb2.SimpleResponse.FromString, + _registered_method=True, ) greenlet = group.spawn(self._run_client, UnaryCallWithSleep) # release loop so that greenlet can take control diff --git a/src/python/grpcio_tests/tests_py3_only/unit/_leak_test.py b/src/python/grpcio_tests/tests_py3_only/unit/_leak_test.py index c917bd10521..06fa6466014 100644 --- a/src/python/grpcio_tests/tests_py3_only/unit/_leak_test.py +++ b/src/python/grpcio_tests/tests_py3_only/unit/_leak_test.py @@ -68,7 +68,10 @@ def _start_a_test_server(): def _perform_an_rpc(address): channel = grpc.insecure_channel(address) - multicallable = channel.unary_unary(_TEST_METHOD) + multicallable = channel.unary_unary( + _TEST_METHOD, + _registered_method=True, + ) response = multicallable(_REQUEST) assert _REQUEST == response diff --git a/src/python/grpcio_tests/tests_py3_only/unit/_simple_stubs_test.py b/src/python/grpcio_tests/tests_py3_only/unit/_simple_stubs_test.py index 771097936f6..adcc1299e9c 100644 --- a/src/python/grpcio_tests/tests_py3_only/unit/_simple_stubs_test.py +++ b/src/python/grpcio_tests/tests_py3_only/unit/_simple_stubs_test.py @@ -193,6 +193,7 @@ class SimpleStubsTest(unittest.TestCase): _UNARY_UNARY, channel_credentials=grpc.experimental.insecure_channel_credentials(), timeout=None, + _registered_method=0, ) self.assertEqual(_REQUEST, response) @@ -205,6 +206,7 @@ class SimpleStubsTest(unittest.TestCase): _UNARY_UNARY, channel_credentials=grpc.local_channel_credentials(), timeout=None, + _registered_method=0, ) self.assertEqual(_REQUEST, response) @@ -213,7 +215,10 @@ class SimpleStubsTest(unittest.TestCase): target = f"localhost:{port}" test_name = inspect.stack()[0][3] args = (_REQUEST, target, _UNARY_UNARY) - kwargs = {"channel_credentials": grpc.local_channel_credentials()} + kwargs = { + "channel_credentials": grpc.local_channel_credentials(), + "_registered_method": True, + } def _invoke(seed: str): run_kwargs = dict(kwargs) @@ -230,6 +235,7 @@ class SimpleStubsTest(unittest.TestCase): target, _UNARY_UNARY, channel_credentials=grpc.local_channel_credentials(), + _registered_method=0, ) self.assert_eventually( lambda: grpc._simple_stubs.ChannelCache.get()._test_only_channel_count() @@ -250,6 +256,7 @@ class SimpleStubsTest(unittest.TestCase): _UNARY_UNARY, options=options, channel_credentials=grpc.local_channel_credentials(), + _registered_method=0, ) self.assert_eventually( lambda: grpc._simple_stubs.ChannelCache.get()._test_only_channel_count() @@ -265,6 +272,7 @@ class SimpleStubsTest(unittest.TestCase): target, _UNARY_STREAM, channel_credentials=grpc.local_channel_credentials(), + _registered_method=0, ): self.assertEqual(_REQUEST, response) @@ -280,6 +288,7 @@ class SimpleStubsTest(unittest.TestCase): target, _STREAM_UNARY, channel_credentials=grpc.local_channel_credentials(), + _registered_method=0, ) self.assertEqual(_REQUEST, response) @@ -295,6 +304,7 @@ class SimpleStubsTest(unittest.TestCase): target, _STREAM_STREAM, channel_credentials=grpc.local_channel_credentials(), + _registered_method=0, ): self.assertEqual(_REQUEST, response) @@ -319,14 +329,22 @@ class SimpleStubsTest(unittest.TestCase): with _server(server_creds) as port: target = f"localhost:{port}" response = grpc.experimental.unary_unary( - _REQUEST, target, _UNARY_UNARY, options=_property_options + _REQUEST, + target, + _UNARY_UNARY, + options=_property_options, + _registered_method=0, ) def test_insecure_sugar(self): with _server(None) as port: target = f"localhost:{port}" response = grpc.experimental.unary_unary( - _REQUEST, target, _UNARY_UNARY, insecure=True + _REQUEST, + target, + _UNARY_UNARY, + insecure=True, + _registered_method=0, ) self.assertEqual(_REQUEST, response) @@ -340,14 +358,24 @@ class SimpleStubsTest(unittest.TestCase): _UNARY_UNARY, insecure=True, channel_credentials=grpc.local_channel_credentials(), + _registered_method=0, ) def test_default_wait_for_ready(self): addr, port, sock = get_socket() sock.close() target = f"{addr}:{port}" - channel = grpc._simple_stubs.ChannelCache.get().get_channel( - target, (), None, True, None + ( + channel, + unused_method_handle, + ) = grpc._simple_stubs.ChannelCache.get().get_channel( + target=target, + options=(), + channel_credentials=None, + insecure=True, + compression=None, + method=_UNARY_UNARY, + _registered_method=True, ) rpc_finished_event = threading.Event() rpc_failed_event = threading.Event() @@ -376,7 +404,12 @@ class SimpleStubsTest(unittest.TestCase): def _send_rpc(): try: response = grpc.experimental.unary_unary( - _REQUEST, target, _UNARY_UNARY, timeout=None, insecure=True + _REQUEST, + target, + _UNARY_UNARY, + timeout=None, + insecure=True, + _registered_method=0, ) rpc_finished_event.set() except Exception as e: @@ -399,6 +432,7 @@ class SimpleStubsTest(unittest.TestCase): target, _BLACK_HOLE, insecure=True, + _registered_method=0, **invocation_args, ) self.assertEqual( From 34be0d84a99a6b313cef48b4c80e14a0d115d864 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Thu, 15 Feb 2024 16:51:37 -0800 Subject: [PATCH 24/37] [Core] Remove wrap_memcpy (#35826) gRPC has been having the `wrap_memcpy` workaround to handle the breaking change of `memcpy` on glibc 2.14 ([ref1](https://www.win.tue.nl/~aeb/linux/misc/gcc-semibug.html), [ref2](https://stackoverflow.com/questions/35656696/explanation-of-memcpy-memmove-glibc-2-14-2-2-5)). This was necessary to build more portable artifacts which are expected to run on most linux distributions and we manged to have https://github.com/grpc/grpc/pull/5007 years ago for this. Since we started to use manylinux2010-based docker images for artifacts, however, this became unnecessary since CentOS 6 has glibc 2.12 which is older than 2.14 so it doesn't have memcpy issue, which is another benefit of using manylinux2010. Superseding https://github.com/grpc/grpc/pull/23385 Changes after the original PR was made; Ruby is using manylinux images (manylinux2014) enabling this change, no more old docker images in our test env. Closes #35826 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/35826 from veblush:del-wrap-memcpy-2 71e6718356b2b6684bb7e977fe66ef39561c05a7 PiperOrigin-RevId: 607499060 --- BUILD | 1 - CMakeLists.txt | 1 - Makefile | 1 - Package.swift | 1 - build_autogenerated.yaml | 1 - config.m4 | 1 - config.w32 | 1 - gRPC-Core.podspec | 1 - grpc.gemspec | 1 - grpc.gyp | 1 - package.xml | 1 - src/core/lib/gpr/wrap_memcpy.cc | 43 ------------------- src/python/grpcio/grpc_core_dependencies.py | 1 - .../observability_lib_deps.py | 1 - src/ruby/ext/grpc/extconf.rb | 1 - tools/doxygen/Doxyfile.c++.internal | 1 - tools/doxygen/Doxyfile.core.internal | 1 - 17 files changed, 59 deletions(-) delete mode 100644 src/core/lib/gpr/wrap_memcpy.cc diff --git a/BUILD b/BUILD index 236a89ec982..e2100794407 100644 --- a/BUILD +++ b/BUILD @@ -706,7 +706,6 @@ grpc_cc_library( "//src/core:lib/gpr/windows/sync.cc", "//src/core:lib/gpr/windows/time.cc", "//src/core:lib/gpr/windows/tmpfile.cc", - "//src/core:lib/gpr/wrap_memcpy.cc", "//src/core:lib/gprpp/crash.cc", "//src/core:lib/gprpp/fork.cc", "//src/core:lib/gprpp/host_port.cc", diff --git a/CMakeLists.txt b/CMakeLists.txt index 88429d02709..6b528b52ec9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1641,7 +1641,6 @@ add_library(gpr src/core/lib/gpr/windows/sync.cc src/core/lib/gpr/windows/time.cc src/core/lib/gpr/windows/tmpfile.cc - src/core/lib/gpr/wrap_memcpy.cc src/core/lib/gprpp/crash.cc src/core/lib/gprpp/examine_stack.cc src/core/lib/gprpp/fork.cc diff --git a/Makefile b/Makefile index 9fe4e2b80b1..f5c020f2582 100644 --- a/Makefile +++ b/Makefile @@ -864,7 +864,6 @@ LIBGPR_SRC = \ src/core/lib/gpr/windows/sync.cc \ src/core/lib/gpr/windows/time.cc \ src/core/lib/gpr/windows/tmpfile.cc \ - src/core/lib/gpr/wrap_memcpy.cc \ src/core/lib/gprpp/crash.cc \ src/core/lib/gprpp/examine_stack.cc \ src/core/lib/gprpp/fork.cc \ diff --git a/Package.swift b/Package.swift index 60ef86d31a9..03b9b6f26b8 100644 --- a/Package.swift +++ b/Package.swift @@ -1333,7 +1333,6 @@ let package = Package( "src/core/lib/gpr/windows/sync.cc", "src/core/lib/gpr/windows/time.cc", "src/core/lib/gpr/windows/tmpfile.cc", - "src/core/lib/gpr/wrap_memcpy.cc", "src/core/lib/gprpp/atomic_utils.h", "src/core/lib/gprpp/bitset.h", "src/core/lib/gprpp/chunked_vector.h", diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index 452e24b9d99..65c275a8246 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -107,7 +107,6 @@ libs: - src/core/lib/gpr/windows/sync.cc - src/core/lib/gpr/windows/time.cc - src/core/lib/gpr/windows/tmpfile.cc - - src/core/lib/gpr/wrap_memcpy.cc - src/core/lib/gprpp/crash.cc - src/core/lib/gprpp/examine_stack.cc - src/core/lib/gprpp/fork.cc diff --git a/config.m4 b/config.m4 index 97c6f517823..35a684dacec 100644 --- a/config.m4 +++ b/config.m4 @@ -567,7 +567,6 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/gpr/windows/sync.cc \ src/core/lib/gpr/windows/time.cc \ src/core/lib/gpr/windows/tmpfile.cc \ - src/core/lib/gpr/wrap_memcpy.cc \ src/core/lib/gprpp/crash.cc \ src/core/lib/gprpp/examine_stack.cc \ src/core/lib/gprpp/fork.cc \ diff --git a/config.w32 b/config.w32 index a07a13e2a2a..c87b296783c 100644 --- a/config.w32 +++ b/config.w32 @@ -532,7 +532,6 @@ if (PHP_GRPC != "no") { "src\\core\\lib\\gpr\\windows\\sync.cc " + "src\\core\\lib\\gpr\\windows\\time.cc " + "src\\core\\lib\\gpr\\windows\\tmpfile.cc " + - "src\\core\\lib\\gpr\\wrap_memcpy.cc " + "src\\core\\lib\\gprpp\\crash.cc " + "src\\core\\lib\\gprpp\\examine_stack.cc " + "src\\core\\lib\\gprpp\\fork.cc " + diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 3e6029dc38e..f67f53c44b1 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -1446,7 +1446,6 @@ Pod::Spec.new do |s| 'src/core/lib/gpr/windows/sync.cc', 'src/core/lib/gpr/windows/time.cc', 'src/core/lib/gpr/windows/tmpfile.cc', - 'src/core/lib/gpr/wrap_memcpy.cc', 'src/core/lib/gprpp/atomic_utils.h', 'src/core/lib/gprpp/bitset.h', 'src/core/lib/gprpp/chunked_vector.h', diff --git a/grpc.gemspec b/grpc.gemspec index cf9045b5bc2..a991d744105 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -1339,7 +1339,6 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/gpr/windows/sync.cc ) s.files += %w( src/core/lib/gpr/windows/time.cc ) s.files += %w( src/core/lib/gpr/windows/tmpfile.cc ) - s.files += %w( src/core/lib/gpr/wrap_memcpy.cc ) s.files += %w( src/core/lib/gprpp/atomic_utils.h ) s.files += %w( src/core/lib/gprpp/bitset.h ) s.files += %w( src/core/lib/gprpp/chunked_vector.h ) diff --git a/grpc.gyp b/grpc.gyp index e3171fcf7a1..20b89da75f1 100644 --- a/grpc.gyp +++ b/grpc.gyp @@ -227,7 +227,6 @@ 'src/core/lib/gpr/windows/sync.cc', 'src/core/lib/gpr/windows/time.cc', 'src/core/lib/gpr/windows/tmpfile.cc', - 'src/core/lib/gpr/wrap_memcpy.cc', 'src/core/lib/gprpp/crash.cc', 'src/core/lib/gprpp/examine_stack.cc', 'src/core/lib/gprpp/fork.cc', diff --git a/package.xml b/package.xml index a0dde5572de..7c2c650fb5d 100644 --- a/package.xml +++ b/package.xml @@ -1321,7 +1321,6 @@ - diff --git a/src/core/lib/gpr/wrap_memcpy.cc b/src/core/lib/gpr/wrap_memcpy.cc deleted file mode 100644 index 0bbf703e613..00000000000 --- a/src/core/lib/gpr/wrap_memcpy.cc +++ /dev/null @@ -1,43 +0,0 @@ -// -// -// Copyright 2016 gRPC authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// - -#include - -#include - -// Provide a wrapped memcpy for targets that need to be backwards -// compatible with older libc's. -// -// Enable by setting LDFLAGS=-Wl,-wrap,memcpy when linking. -// - -extern "C" { -#ifdef __linux__ -#if defined(__x86_64__) && !defined(GPR_MUSL_LIBC_COMPAT) && \ - !defined(__ANDROID__) -__asm__(".symver memcpy,memcpy@GLIBC_2.2.5"); -void* __wrap_memcpy(void* destination, const void* source, size_t num) { - return memcpy(destination, source, num); -} -#else // !__x86_64__ -void* __wrap_memcpy(void* destination, const void* source, size_t num) { - return memmove(destination, source, num); -} -#endif -#endif -} diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 28823ac0bf2..ef280852032 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -541,7 +541,6 @@ CORE_SOURCE_FILES = [ 'src/core/lib/gpr/windows/sync.cc', 'src/core/lib/gpr/windows/time.cc', 'src/core/lib/gpr/windows/tmpfile.cc', - 'src/core/lib/gpr/wrap_memcpy.cc', 'src/core/lib/gprpp/crash.cc', 'src/core/lib/gprpp/examine_stack.cc', 'src/core/lib/gprpp/fork.cc', diff --git a/src/python/grpcio_observability/observability_lib_deps.py b/src/python/grpcio_observability/observability_lib_deps.py index 14fd1c9034b..eb117797788 100644 --- a/src/python/grpcio_observability/observability_lib_deps.py +++ b/src/python/grpcio_observability/observability_lib_deps.py @@ -46,7 +46,6 @@ CC_FILES=[ 'grpc_root/src/core/lib/gpr/windows/sync.cc', 'grpc_root/src/core/lib/gpr/windows/time.cc', 'grpc_root/src/core/lib/gpr/windows/tmpfile.cc', - 'grpc_root/src/core/lib/gpr/wrap_memcpy.cc', 'grpc_root/src/core/lib/gprpp/crash.cc', 'grpc_root/src/core/lib/gprpp/examine_stack.cc', 'grpc_root/src/core/lib/gprpp/fork.cc', diff --git a/src/ruby/ext/grpc/extconf.rb b/src/ruby/ext/grpc/extconf.rb index 502793d9198..18e8a919c53 100644 --- a/src/ruby/ext/grpc/extconf.rb +++ b/src/ruby/ext/grpc/extconf.rb @@ -192,7 +192,6 @@ if grpc_config == 'dbg' $CFLAGS << ' -O0' end -$LDFLAGS << ' -Wl,-wrap,memcpy' if linux # Do not statically link standard libraries on TruffleRuby as this does not work when compiling to bitcode if linux && RUBY_ENGINE != 'truffleruby' $LDFLAGS << ' -static-libgcc -static-libstdc++' diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 27f47e275a9..c19780fd743 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -2338,7 +2338,6 @@ src/core/lib/gpr/windows/string_util.cc \ src/core/lib/gpr/windows/sync.cc \ src/core/lib/gpr/windows/time.cc \ src/core/lib/gpr/windows/tmpfile.cc \ -src/core/lib/gpr/wrap_memcpy.cc \ src/core/lib/gprpp/atomic_utils.h \ src/core/lib/gprpp/bitset.h \ src/core/lib/gprpp/chunked_vector.h \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 9ee3aeff6a1..35a1d5fb44f 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -2111,7 +2111,6 @@ src/core/lib/gpr/windows/string_util.cc \ src/core/lib/gpr/windows/sync.cc \ src/core/lib/gpr/windows/time.cc \ src/core/lib/gpr/windows/tmpfile.cc \ -src/core/lib/gpr/wrap_memcpy.cc \ src/core/lib/gprpp/README.md \ src/core/lib/gprpp/atomic_utils.h \ src/core/lib/gprpp/bitset.h \ From 310770d61d3b25d358d877a074bebf6ae85549d0 Mon Sep 17 00:00:00 2001 From: Gregory Cooke Date: Fri, 16 Feb 2024 08:42:49 -0800 Subject: [PATCH 25/37] [Security - CrlProvider] Use a better mechanism for Crl Lookup and add some verification helpers (#35641) This PR does 2 distinct things, I can unbundle them if desired 1) Add functions in `ssl_transport_security_utils` and associated tests that will eventually be used for additional Crl validation (the logic of actually doing this will be in a future PR), so other than the tests these fns are currently unused. 2) Remove the use of `X509_NAME_oneline` - it is not a guaranteed stable way to get the issuer name for lookups. Instead, use the DER encoding via `i2d_X509_NAME` - the results in a non-human readable string that is stable for lookup, and necessitated some change to the CrlProvider test code that previously used a human readable string for this value. Neither should result in behavior changes. Closes #35641 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/35641 from gtcooke94:CrlRefactor 2b6f63717c3f6495740c343795588e5521ad8586 PiperOrigin-RevId: 607701254 --- BUILD | 1 + CMakeLists.txt | 2 + build_autogenerated.yaml | 6 +- .../credentials/tls/grpc_tls_crl_provider.cc | 27 ++- src/core/tsi/ssl_transport_security.cc | 9 +- src/core/tsi/ssl_transport_security_utils.cc | 66 ++++++ src/core/tsi/ssl_transport_security_utils.h | 18 ++ test/core/security/BUILD | 2 + .../security/grpc_tls_crl_provider_test.cc | 67 ++++-- test/core/tsi/BUILD | 11 + .../tsi/ssl_transport_security_utils_test.cc | 198 +++++++++++++++++- test/core/tsi/test_creds/crl_data/BUILD | 1 + test/core/tsi/test_creds/crl_data/README | 5 + test/core/tsi/test_creds/crl_data/crls/BUILD | 2 + .../crl_data/crls/invalid_content.crl | 15 ++ .../crl_data/crls/invalid_signature.crl | 15 ++ test/core/tsi/test_creds/crl_data/evil_ca.key | 28 +++ test/core/tsi/test_creds/crl_data/evil_ca.pem | 21 ++ .../tsi/test_creds/crl_data/evil_ca_gen.sh | 18 ++ test/core/tsi/transport_security_test_lib.cc | 24 +++ test/core/tsi/transport_security_test_lib.h | 8 + 21 files changed, 503 insertions(+), 41 deletions(-) create mode 100644 test/core/tsi/test_creds/crl_data/crls/invalid_content.crl create mode 100644 test/core/tsi/test_creds/crl_data/crls/invalid_signature.crl create mode 100644 test/core/tsi/test_creds/crl_data/evil_ca.key create mode 100644 test/core/tsi/test_creds/crl_data/evil_ca.pem create mode 100644 test/core/tsi/test_creds/crl_data/evil_ca_gen.sh diff --git a/BUILD b/BUILD index e2100794407..ca76d4c83b4 100644 --- a/BUILD +++ b/BUILD @@ -3561,6 +3561,7 @@ grpc_cc_library( external_deps = [ "absl/base:core_headers", "absl/status", + "absl/status:statusor", "absl/strings", "libcrypto", "libssl", diff --git a/CMakeLists.txt b/CMakeLists.txt index 6b528b52ec9..1c6288b3140 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15930,6 +15930,7 @@ add_executable(grpc_tls_crl_provider_test test/core/event_engine/event_engine_test_utils.cc test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.cc test/core/security/grpc_tls_crl_provider_test.cc + test/core/tsi/transport_security_test_lib.cc ) if(WIN32 AND MSVC) if(BUILD_SHARED_LIBS) @@ -28087,6 +28088,7 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(ssl_transport_security_utils_test test/core/tsi/ssl_transport_security_utils_test.cc + test/core/tsi/transport_security_test_lib.cc ) if(WIN32 AND MSVC) if(BUILD_SHARED_LIBS) diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index 65c275a8246..3d0de3e0b37 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -10772,12 +10772,14 @@ targets: headers: - test/core/event_engine/event_engine_test_utils.h - test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.h + - test/core/tsi/transport_security_test_lib.h src: - test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.proto - test/core/util/fuzz_config_vars.proto - test/core/event_engine/event_engine_test_utils.cc - test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.cc - test/core/security/grpc_tls_crl_provider_test.cc + - test/core/tsi/transport_security_test_lib.cc deps: - gtest - protobuf @@ -18037,9 +18039,11 @@ targets: gtest: true build: test language: c++ - headers: [] + headers: + - test/core/tsi/transport_security_test_lib.h src: - test/core/tsi/ssl_transport_security_utils_test.cc + - test/core/tsi/transport_security_test_lib.cc deps: - gtest - grpc_test_util diff --git a/src/core/lib/security/credentials/tls/grpc_tls_crl_provider.cc b/src/core/lib/security/credentials/tls/grpc_tls_crl_provider.cc index f9c0868f7c1..f8adf5f12cc 100644 --- a/src/core/lib/security/credentials/tls/grpc_tls_crl_provider.cc +++ b/src/core/lib/security/credentials/tls/grpc_tls_crl_provider.cc @@ -53,15 +53,22 @@ namespace grpc_core { namespace experimental { namespace { -std::string IssuerFromCrl(X509_CRL* crl) { +// TODO(gtcooke94) Move ssl_transport_security_utils to it's own BUILD target +// and add this to it. +absl::StatusOr IssuerFromCrl(X509_CRL* crl) { if (crl == nullptr) { - return ""; + return absl::InvalidArgumentError("crl cannot be null"); } - char* buf = X509_NAME_oneline(X509_CRL_get_issuer(crl), nullptr, 0); - std::string ret; - if (buf != nullptr) { - ret = buf; + X509_NAME* issuer = X509_CRL_get_issuer(crl); + if (issuer == nullptr) { + return absl::InvalidArgumentError("crl cannot have null issuer"); } + unsigned char* buf = nullptr; + int len = i2d_X509_NAME(issuer, &buf); + if (len < 0 || buf == nullptr) { + return absl::InvalidArgumentError("crl cannot have null issuer"); + } + std::string ret(reinterpret_cast(buf), len); OPENSSL_free(buf); return ret; } @@ -103,11 +110,11 @@ absl::StatusOr> Crl::Parse(absl::string_view crl_string) { } absl::StatusOr> CrlImpl::Create(X509_CRL* crl) { - std::string issuer = IssuerFromCrl(crl); - if (issuer.empty()) { - return absl::InvalidArgumentError("Issuer of crl cannot be empty"); + absl::StatusOr issuer = IssuerFromCrl(crl); + if (!issuer.ok()) { + return issuer.status(); } - return std::make_unique(crl, issuer); + return std::make_unique(crl, *issuer); } CrlImpl::~CrlImpl() { X509_CRL_free(crl_); } diff --git a/src/core/tsi/ssl_transport_security.cc b/src/core/tsi/ssl_transport_security.cc index 7c6ba7c5ecb..ef79db1c0a2 100644 --- a/src/core/tsi/ssl_transport_security.cc +++ b/src/core/tsi/ssl_transport_security.cc @@ -1004,15 +1004,14 @@ static int GetCrlFromProvider(X509_STORE_CTX* ctx, X509_CRL** crl_out, auto* provider = static_cast( SSL_CTX_get_ex_data(ssl_ctx, g_ssl_ctx_ex_crl_provider_index)); - char* buf = X509_NAME_oneline(X509_get_issuer_name(cert), nullptr, 0); - if (buf == nullptr) { - gpr_log(GPR_ERROR, "Certificate has null issuer, cannot do CRL lookup"); + absl::StatusOr issuer_name = grpc_core::IssuerFromCert(cert); + if (!issuer_name.ok()) { + gpr_log(GPR_ERROR, "Could not get certificate issuer name"); return 0; } - grpc_core::experimental::CertificateInfoImpl cert_impl(buf); + grpc_core::experimental::CertificateInfoImpl cert_impl(*issuer_name); std::shared_ptr internal_crl = provider->GetCrl(cert_impl); - OPENSSL_free(buf); // There wasn't a CRL found in the provider. Returning 0 will end up causing // OpenSSL to return X509_V_ERR_UNABLE_TO_GET_CRL. We then catch that error // and behave how we want for a missing CRL. diff --git a/src/core/tsi/ssl_transport_security_utils.cc b/src/core/tsi/ssl_transport_security_utils.cc index 7d258046600..07ea0e0cc65 100644 --- a/src/core/tsi/ssl_transport_security_utils.cc +++ b/src/core/tsi/ssl_transport_security_utils.cc @@ -23,6 +23,10 @@ #include #include #include +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" #include "src/core/tsi/transport_security_interface.h" @@ -247,4 +251,66 @@ tsi_result SslProtectorUnprotect(const unsigned char* protected_frames_bytes, return result; } +bool VerifyCrlSignature(X509_CRL* crl, X509* issuer) { + if (issuer == nullptr || crl == nullptr) { + return false; + } + EVP_PKEY* ikey = X509_get_pubkey(issuer); + if (ikey == nullptr) { + // Can't verify signature because we couldn't get the pubkey, fail the + // check. + EVP_PKEY_free(ikey); + return false; + } + bool ret = X509_CRL_verify(crl, ikey) == 1; + EVP_PKEY_free(ikey); + return ret; +} + +bool VerifyCrlCertIssuerNamesMatch(X509_CRL* crl, X509* cert) { + if (cert == nullptr || crl == nullptr) { + return false; + } + X509_NAME* cert_issuer_name = X509_get_issuer_name(cert); + if (cert == nullptr) { + return false; + } + X509_NAME* crl_issuer_name = X509_CRL_get_issuer(crl); + if (crl_issuer_name == nullptr) { + return false; + } + return X509_NAME_cmp(cert_issuer_name, crl_issuer_name) == 0; +} + +bool HasCrlSignBit(X509* cert) { + if (cert == nullptr) { + return false; + } + // X509_get_key_usage was introduced in 1.1.1 + // A missing key usage extension means all key usages are valid. +#if OPENSSL_VERSION_NUMBER < 0x10100000 + if (!cert->ex_flags & EXFLAG_KUSAGE) { + return true; + } + return cert->ex_kusage & KU_CRL_SIGN; +#else + return (X509_get_key_usage(cert) & KU_CRL_SIGN) != 0; +#endif // OPENSSL_VERSION_NUMBER < 0x10100000 +} + +absl::StatusOr IssuerFromCert(X509* cert) { + if (cert == nullptr) { + return absl::InvalidArgumentError("cert cannot be null"); + } + X509_NAME* issuer = X509_get_issuer_name(cert); + unsigned char* buf = nullptr; + int len = i2d_X509_NAME(issuer, &buf); + if (len < 0 || buf == nullptr) { + return absl::InvalidArgumentError("could not read issuer name from cert"); + } + std::string ret(reinterpret_cast(buf), len); + OPENSSL_free(buf); + return ret; +} + } // namespace grpc_core diff --git a/src/core/tsi/ssl_transport_security_utils.h b/src/core/tsi/ssl_transport_security_utils.h index 14b38e644d1..22cacc84fbc 100644 --- a/src/core/tsi/ssl_transport_security_utils.h +++ b/src/core/tsi/ssl_transport_security_utils.h @@ -23,6 +23,8 @@ #include +#include "absl/status/status.h" +#include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include @@ -142,6 +144,22 @@ tsi_result SslProtectorUnprotect(const unsigned char* protected_frames_bytes, unsigned char* unprotected_bytes, size_t* unprotected_bytes_size); +// Verifies that `crl` was signed by `issuer. +// return: true if valid, false otherwise. +bool VerifyCrlSignature(X509_CRL* crl, X509* issuer); + +// Verifies the CRL issuer and certificate issuer name match. +// return: true if equal, false if not. +bool VerifyCrlCertIssuerNamesMatch(X509_CRL* crl, X509* cert); + +// Verifies the certificate in question has the cRLSign bit present. +// return: true if cRLSign bit is present, false otherwise. +bool HasCrlSignBit(X509* cert); + +// Gets a stable representation of the issuer name from an X509 certificate. +// return: a std::string of the DER encoding of the X509_NAME issuer name. +absl::StatusOr IssuerFromCert(X509* cert); + } // namespace grpc_core #endif // GRPC_SRC_CORE_TSI_SSL_TRANSPORT_SECURITY_UTILS_H diff --git a/test/core/security/BUILD b/test/core/security/BUILD index 6b7f7bf4899..2899bc9f523 100644 --- a/test/core/security/BUILD +++ b/test/core/security/BUILD @@ -569,6 +569,7 @@ grpc_cc_test( srcs = ["grpc_tls_crl_provider_test.cc"], data = [ "//test/core/tsi/test_creds/crl_data:ca.pem", + "//test/core/tsi/test_creds/crl_data:intermediate_ca.pem", "//test/core/tsi/test_creds/crl_data/crls:ab06acdd.r0", "//test/core/tsi/test_creds/crl_data/crls:b9322cac.r0", "//test/core/tsi/test_creds/crl_data/crls:current.crl", @@ -585,6 +586,7 @@ grpc_cc_test( "//test/core/event_engine:event_engine_test_utils", "//test/core/event_engine/fuzzing_event_engine", "//test/core/event_engine/fuzzing_event_engine:fuzzing_event_engine_proto", + "//test/core/tsi:transport_security_test_lib", "//test/core/util:fuzz_config_vars_proto", "//test/core/util:grpc_test_util", ], diff --git a/test/core/security/grpc_tls_crl_provider_test.cc b/test/core/security/grpc_tls_crl_provider_test.cc index f47028b160b..76cc0bdf34d 100644 --- a/test/core/security/grpc_tls_crl_provider_test.cc +++ b/test/core/security/grpc_tls_crl_provider_test.cc @@ -40,16 +40,15 @@ #include "test/core/event_engine/event_engine_test_utils.h" #include "test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.h" #include "test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.pb.h" +#include "test/core/tsi/transport_security_test_lib.h" #include "test/core/util/test_config.h" #include "test/core/util/tls_utils.h" static constexpr absl::string_view kCrlPath = "test/core/tsi/test_creds/crl_data/crls/current.crl"; static constexpr absl::string_view kCrlName = "current.crl"; -static constexpr absl::string_view kCrlIssuer = - "/C=AU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=testca"; -static constexpr absl::string_view kCrlIntermediateIssuer = - "/CN=intermediatecert.example.com"; +static constexpr absl::string_view kCrlIntermediateIssuerPath = + "test/core/tsi/test_creds/crl_data/intermediate_ca.pem"; static constexpr absl::string_view kCrlDirectory = "test/core/tsi/test_creds/crl_data/crls"; static constexpr absl::string_view kRootCert = @@ -88,9 +87,35 @@ class FakeDirectoryReader : public DirectoryReader { std::vector(); }; -class DirectoryReloaderCrlProviderTest : public ::testing::Test { +class CrlProviderTest : public ::testing::Test { public: void SetUp() override { + std::string pem_cert = GetFileContents(kRootCert.data()); + X509* issuer = ReadPemCert(pem_cert); + auto base_crl_issuer = IssuerFromCert(issuer); + ASSERT_EQ(base_crl_issuer.status(), absl::OkStatus()); + base_crl_issuer_ = *base_crl_issuer; + std::string intermediate_string = + GetFileContents(kCrlIntermediateIssuerPath.data()); + X509* intermediate_issuer = ReadPemCert(intermediate_string); + auto intermediate_crl_issuer = IssuerFromCert(intermediate_issuer); + ASSERT_EQ(intermediate_crl_issuer.status(), absl::OkStatus()); + intermediate_crl_issuer_ = *intermediate_crl_issuer; + X509_free(issuer); + X509_free(intermediate_issuer); + } + + void TearDown() override {} + + protected: + std::string base_crl_issuer_; + std::string intermediate_crl_issuer_; +}; + +class DirectoryReloaderCrlProviderTest : public CrlProviderTest { + public: + void SetUp() override { + CrlProviderTest::SetUp(); event_engine_ = std::make_shared( grpc_event_engine::experimental::FuzzingEventEngine::Options(), @@ -140,15 +165,15 @@ class DirectoryReloaderCrlProviderTest : public ::testing::Test { event_engine_; }; -TEST(CrlProviderTest, CanParseCrl) { +TEST_F(CrlProviderTest, CanParseCrl) { std::string crl_string = GetFileContents(kCrlPath.data()); absl::StatusOr> crl = Crl::Parse(crl_string); ASSERT_TRUE(crl.ok()) << crl.status(); ASSERT_NE(*crl, nullptr); - EXPECT_EQ((*crl)->Issuer(), kCrlIssuer); + EXPECT_EQ((*crl)->Issuer(), base_crl_issuer_); } -TEST(CrlProviderTest, InvalidFile) { +TEST_F(CrlProviderTest, InvalidFile) { std::string crl_string = "INVALID CRL FILE"; absl::StatusOr> crl = Crl::Parse(crl_string); EXPECT_EQ(crl.status(), @@ -156,18 +181,18 @@ TEST(CrlProviderTest, InvalidFile) { "Conversion from PEM string to X509 CRL failed.")); } -TEST(CrlProviderTest, StaticCrlProviderLookup) { +TEST_F(CrlProviderTest, StaticCrlProviderLookup) { std::vector crl_strings = {GetFileContents(kCrlPath.data())}; absl::StatusOr> provider = experimental::CreateStaticCrlProvider(crl_strings); ASSERT_TRUE(provider.ok()) << provider.status(); - CertificateInfoImpl cert(kCrlIssuer); + CertificateInfoImpl cert(base_crl_issuer_); auto crl = (*provider)->GetCrl(cert); ASSERT_NE(crl, nullptr); - EXPECT_EQ(crl->Issuer(), kCrlIssuer); + EXPECT_EQ(crl->Issuer(), base_crl_issuer_); } -TEST(CrlProviderTest, StaticCrlProviderLookupIssuerNotFound) { +TEST_F(CrlProviderTest, StaticCrlProviderLookupIssuerNotFound) { std::vector crl_strings = {GetFileContents(kCrlPath.data())}; absl::StatusOr> provider = experimental::CreateStaticCrlProvider(crl_strings); @@ -181,14 +206,14 @@ TEST_F(DirectoryReloaderCrlProviderTest, CrlLookupGood) { auto provider = CreateCrlProvider(kCrlDirectory, std::chrono::seconds(60), nullptr); ASSERT_TRUE(provider.ok()) << provider.status(); - CertificateInfoImpl cert(kCrlIssuer); + CertificateInfoImpl cert(base_crl_issuer_); auto crl = (*provider)->GetCrl(cert); ASSERT_NE(crl, nullptr); - EXPECT_EQ(crl->Issuer(), kCrlIssuer); - CertificateInfoImpl intermediate(kCrlIntermediateIssuer); + EXPECT_EQ(crl->Issuer(), base_crl_issuer_); + CertificateInfoImpl intermediate(intermediate_crl_issuer_); auto intermediate_crl = (*provider)->GetCrl(intermediate); ASSERT_NE(intermediate_crl, nullptr); - EXPECT_EQ(intermediate_crl->Issuer(), kCrlIntermediateIssuer); + EXPECT_EQ(intermediate_crl->Issuer(), intermediate_crl_issuer_); } TEST_F(DirectoryReloaderCrlProviderTest, CrlLookupMissingIssuer) { @@ -204,7 +229,7 @@ TEST_F(DirectoryReloaderCrlProviderTest, ReloadsAndDeletes) { const std::chrono::seconds kRefreshDuration(60); auto provider = CreateCrlProvider(kRefreshDuration, nullptr); ASSERT_TRUE(provider.ok()) << provider.status(); - CertificateInfoImpl cert(kCrlIssuer); + CertificateInfoImpl cert(base_crl_issuer_); auto should_be_no_crl = (*provider)->GetCrl(cert); ASSERT_EQ(should_be_no_crl, nullptr); // Give the provider files to find in the directory @@ -212,7 +237,7 @@ TEST_F(DirectoryReloaderCrlProviderTest, ReloadsAndDeletes) { event_engine_->TickForDuration(kRefreshDuration); auto crl = (*provider)->GetCrl(cert); ASSERT_NE(crl, nullptr); - EXPECT_EQ(crl->Issuer(), kCrlIssuer); + EXPECT_EQ(crl->Issuer(), base_crl_issuer_); // Now we won't see any files in our directory directory_reader_->SetFilesInDirectory({}); event_engine_->TickForDuration(kRefreshDuration); @@ -229,10 +254,10 @@ TEST_F(DirectoryReloaderCrlProviderTest, WithCorruption) { auto provider = CreateCrlProvider(kRefreshDuration, std::move(reload_error_callback)); ASSERT_TRUE(provider.ok()) << provider.status(); - CertificateInfoImpl cert(kCrlIssuer); + CertificateInfoImpl cert(base_crl_issuer_); auto crl = (*provider)->GetCrl(cert); ASSERT_NE(crl, nullptr); - EXPECT_EQ(crl->Issuer(), kCrlIssuer); + EXPECT_EQ(crl->Issuer(), base_crl_issuer_); EXPECT_EQ(reload_errors.size(), 0); // Point the provider at a non-crl file so loading fails // Should result in the CRL Reloader keeping the old CRL data @@ -240,7 +265,7 @@ TEST_F(DirectoryReloaderCrlProviderTest, WithCorruption) { event_engine_->TickForDuration(kRefreshDuration); auto crl_post_update = (*provider)->GetCrl(cert); ASSERT_NE(crl_post_update, nullptr); - EXPECT_EQ(crl_post_update->Issuer(), kCrlIssuer); + EXPECT_EQ(crl_post_update->Issuer(), base_crl_issuer_); EXPECT_EQ(reload_errors.size(), 1); } diff --git a/test/core/tsi/BUILD b/test/core/tsi/BUILD index 372ba9a3747..be8a6f3de82 100644 --- a/test/core/tsi/BUILD +++ b/test/core/tsi/BUILD @@ -64,12 +64,23 @@ grpc_cc_test( grpc_cc_test( name = "ssl_transport_security_utils_test", srcs = ["ssl_transport_security_utils_test.cc"], + data = [ + "//test/core/tsi/test_creds/crl_data:ca.pem", + "//test/core/tsi/test_creds/crl_data:evil_ca.pem", + "//test/core/tsi/test_creds/crl_data:intermediate_ca.pem", + "//test/core/tsi/test_creds/crl_data:leaf_signed_by_intermediate.pem", + "//test/core/tsi/test_creds/crl_data/crls:current.crl", + "//test/core/tsi/test_creds/crl_data/crls:intermediate.crl", + "//test/core/tsi/test_creds/crl_data/crls:invalid_content.crl", + "//test/core/tsi/test_creds/crl_data/crls:invalid_signature.crl", + ], external_deps = ["gtest"], language = "C++", tags = ["no_windows"], deps = [ "//:gpr", "//:grpc", + "//test/core/tsi:transport_security_test_lib", "//test/core/util:grpc_test_util", ], ) diff --git a/test/core/tsi/ssl_transport_security_utils_test.cc b/test/core/tsi/ssl_transport_security_utils_test.cc index 00e14627a14..07e65e55bc3 100644 --- a/test/core/tsi/ssl_transport_security_utils_test.cc +++ b/test/core/tsi/ssl_transport_security_utils_test.cc @@ -25,6 +25,7 @@ #include #include #include +#include #include "absl/status/status.h" #include "absl/status/statusor.h" @@ -32,13 +33,30 @@ #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" +#include "src/core/lib/gprpp/load_file.h" +#include "src/core/lib/slice/slice.h" #include "src/core/tsi/transport_security.h" #include "src/core/tsi/transport_security_interface.h" +#include "test/core/tsi/transport_security_test_lib.h" #include "test/core/util/test_config.h" namespace grpc_core { namespace testing { +const char* kValidCrl = "test/core/tsi/test_creds/crl_data/crls/current.crl"; +const char* kCrlIssuer = "test/core/tsi/test_creds/crl_data/ca.pem"; +const char* kModifiedSignature = + "test/core/tsi/test_creds/crl_data/crls/invalid_signature.crl"; +const char* kModifiedContent = + "test/core/tsi/test_creds/crl_data/crls/invalid_content.crl"; +const char* kIntermediateCrl = + "test/core/tsi/test_creds/crl_data/crls/intermediate.crl"; +const char* kIntermediateCrlIssuer = + "test/core/tsi/test_creds/crl_data/intermediate_ca.pem"; +const char* kLeafCert = + "test/core/tsi/test_creds/crl_data/leaf_signed_by_intermediate.pem"; +const char* kEvilCa = "test/core/tsi/test_creds/crl_data/evil_ca.pem"; + using ::testing::ContainerEq; using ::testing::NotNull; using ::testing::TestWithParam; @@ -316,8 +334,8 @@ TEST_P(FlowTest, &protected_output_frames_size), tsi_result::TSI_OK); - // If |GetParam().plaintext_size| is larger than the inner client_buffer size - // (kMaxPlaintextBytesPerTlsRecord), then |Protect| will copy up to + // If |GetParam().plaintext_size| is larger than the inner client_buffer + // size (kMaxPlaintextBytesPerTlsRecord), then |Protect| will copy up to // |kMaxPlaintextBytesPerTlsRecord| bytes and output the protected // frame. Otherwise we need to manually flush the copied data in order // to get the protected frame. @@ -378,8 +396,8 @@ TEST_P(FlowTest, &protected_output_frames_size), tsi_result::TSI_OK); - // If |GetParam().plaintext_size| is larger than the inner server_buffer size - // (kMaxPlaintextBytesPerTlsRecord), then |Protect| will copy up to + // If |GetParam().plaintext_size| is larger than the inner server_buffer + // size (kMaxPlaintextBytesPerTlsRecord), then |Protect| will copy up to // |kMaxPlaintextBytesPerTlsRecord| bytes and output the protected // frame. Otherwise we need to manually flush the copied data in order // to get the protected frame. @@ -429,6 +447,178 @@ INSTANTIATE_TEST_SUITE_P(FrameProtectorUtil, FlowTest, #endif // OPENSSL_IS_BORINGSSL +class CrlUtils : public ::testing::Test { + public: + void SetUp() override { + absl::StatusOr root_crl = LoadFile(kValidCrl, false); + ASSERT_EQ(root_crl.status(), absl::OkStatus()) << root_crl.status(); + root_crl_ = ReadCrl(root_crl->as_string_view()); + absl::StatusOr intermediate_crl = LoadFile(kIntermediateCrl, false); + ASSERT_EQ(intermediate_crl.status(), absl::OkStatus()) + << intermediate_crl.status(); + intermediate_crl_ = ReadCrl(intermediate_crl->as_string_view()); + absl::StatusOr invalid_signature_crl = + LoadFile(kModifiedSignature, false); + ASSERT_EQ(invalid_signature_crl.status(), absl::OkStatus()) + << invalid_signature_crl.status(); + invalid_signature_crl_ = ReadCrl(invalid_signature_crl->as_string_view()); + + absl::StatusOr root_ca = LoadFile(kCrlIssuer, false); + ASSERT_EQ(root_ca.status(), absl::OkStatus()); + root_ca_ = ReadPemCert(root_ca->as_string_view()); + absl::StatusOr intermediate_ca = + LoadFile(kIntermediateCrlIssuer, false); + ASSERT_EQ(intermediate_ca.status(), absl::OkStatus()); + intermediate_ca_ = ReadPemCert(intermediate_ca->as_string_view()); + absl::StatusOr leaf_cert = LoadFile(kLeafCert, false); + ASSERT_EQ(leaf_cert.status(), absl::OkStatus()); + leaf_cert_ = ReadPemCert(leaf_cert->as_string_view()); + absl::StatusOr evil_ca = LoadFile(kEvilCa, false); + ASSERT_EQ(evil_ca.status(), absl::OkStatus()); + evil_ca_ = ReadPemCert(evil_ca->as_string_view()); + } + + void TearDown() override { + X509_CRL_free(root_crl_); + X509_CRL_free(intermediate_crl_); + X509_CRL_free(invalid_signature_crl_); + X509_free(root_ca_); + X509_free(intermediate_ca_); + X509_free(leaf_cert_); + X509_free(evil_ca_); + } + + protected: + X509_CRL* root_crl_; + X509_CRL* intermediate_crl_; + X509_CRL* invalid_signature_crl_; + X509* root_ca_; + X509* intermediate_ca_; + X509* leaf_cert_; + X509* evil_ca_; +}; + +TEST_F(CrlUtils, VerifySignatureValid) { + EXPECT_TRUE(VerifyCrlSignature(root_crl_, root_ca_)); +} + +TEST_F(CrlUtils, VerifySignatureIntermediateValid) { + EXPECT_TRUE(VerifyCrlSignature(intermediate_crl_, intermediate_ca_)); +} + +TEST_F(CrlUtils, VerifySignatureModifiedSignature) { + EXPECT_FALSE(VerifyCrlSignature(invalid_signature_crl_, root_ca_)); +} + +TEST_F(CrlUtils, VerifySignatureModifiedContent) { + absl::StatusOr crl_slice = LoadFile(kModifiedContent, false); + ASSERT_EQ(crl_slice.status(), absl::OkStatus()) << crl_slice.status(); + X509_CRL* crl = ReadCrl(crl_slice->as_string_view()); + EXPECT_EQ(crl, nullptr); +} + +TEST_F(CrlUtils, VerifySignatureWrongIssuer) { + EXPECT_FALSE(VerifyCrlSignature(root_crl_, intermediate_ca_)); +} + +TEST_F(CrlUtils, VerifySignatureWrongIssuer2) { + EXPECT_FALSE(VerifyCrlSignature(intermediate_crl_, root_ca_)); +} + +TEST_F(CrlUtils, VerifySignatureNullCrl) { + EXPECT_FALSE(VerifyCrlSignature(nullptr, root_ca_)); +} + +TEST_F(CrlUtils, VerifySignatureNullCert) { + EXPECT_FALSE(VerifyCrlSignature(intermediate_crl_, nullptr)); +} + +TEST_F(CrlUtils, VerifySignatureNullCrlAndCert) { + EXPECT_FALSE(VerifyCrlSignature(nullptr, nullptr)); +} + +TEST_F(CrlUtils, VerifyIssuerNamesMatch) { + EXPECT_TRUE(VerifyCrlCertIssuerNamesMatch(root_crl_, root_ca_)); +} + +TEST_F(CrlUtils, VerifyIssuerNamesDontMatch) { + EXPECT_FALSE(VerifyCrlCertIssuerNamesMatch(root_crl_, leaf_cert_)); +} + +TEST_F(CrlUtils, DuplicatedIssuerNamePassesButSignatureCheckFails) { + // The issuer names will match, but it should fail a signature check + EXPECT_TRUE(VerifyCrlCertIssuerNamesMatch(root_crl_, evil_ca_)); + EXPECT_FALSE(VerifyCrlSignature(root_crl_, evil_ca_)); +} + +TEST_F(CrlUtils, VerifyIssuerNameNullCrl) { + EXPECT_FALSE(VerifyCrlCertIssuerNamesMatch(nullptr, root_ca_)); +} + +TEST_F(CrlUtils, VerifyIssuerNameNullCert) { + EXPECT_FALSE(VerifyCrlCertIssuerNamesMatch(intermediate_crl_, nullptr)); +} + +TEST_F(CrlUtils, VerifyIssuerNameNullCrlAndCert) { + EXPECT_FALSE(VerifyCrlCertIssuerNamesMatch(nullptr, nullptr)); +} + +TEST_F(CrlUtils, HasCrlSignBitExists) { EXPECT_TRUE(HasCrlSignBit(root_ca_)); } + +TEST_F(CrlUtils, HasCrlSignBitMissing) { + EXPECT_FALSE(HasCrlSignBit(leaf_cert_)); +} + +TEST_F(CrlUtils, HasCrlSignBitNullCert) { + EXPECT_FALSE(HasCrlSignBit(nullptr)); +} + +TEST_F(CrlUtils, IssuerFromIntermediateCert) { + auto issuer = IssuerFromCert(intermediate_ca_); + // Build the known name for comparison + unsigned char* buf = nullptr; + X509_NAME* expected_issuer_name = X509_NAME_new(); + ASSERT_TRUE( + X509_NAME_add_entry_by_txt(expected_issuer_name, "C", MBSTRING_ASC, + (const unsigned char*)"AU", -1, -1, 0)); + ASSERT_TRUE(X509_NAME_add_entry_by_txt( + expected_issuer_name, "ST", MBSTRING_ASC, + (const unsigned char*)"Some-State", -1, -1, 0)); + ASSERT_TRUE(X509_NAME_add_entry_by_txt( + expected_issuer_name, "O", MBSTRING_ASC, + (const unsigned char*)"Internet Widgits Pty Ltd", -1, -1, 0)); + ASSERT_TRUE( + X509_NAME_add_entry_by_txt(expected_issuer_name, "CN", MBSTRING_ASC, + (const unsigned char*)"testca", -1, -1, 0)); + int len = i2d_X509_NAME(expected_issuer_name, &buf); + std::string expected_issuer_name_der(reinterpret_cast(buf), len); + OPENSSL_free(buf); + X509_NAME_free(expected_issuer_name); + ASSERT_EQ(issuer.status(), absl::OkStatus()); + EXPECT_EQ(*issuer, expected_issuer_name_der); +} + +TEST_F(CrlUtils, IssuerFromLeaf) { + auto issuer = IssuerFromCert(leaf_cert_); + // Build the known name for comparison + unsigned char* buf = nullptr; + X509_NAME* expected_issuer_name = X509_NAME_new(); + ASSERT_TRUE(X509_NAME_add_entry_by_txt( + expected_issuer_name, "CN", MBSTRING_ASC, + (const unsigned char*)"intermediatecert.example.com", -1, -1, 0)); + int len = i2d_X509_NAME(expected_issuer_name, &buf); + std::string expected_issuer_name_der(reinterpret_cast(buf), len); + OPENSSL_free(buf); + X509_NAME_free(expected_issuer_name); + ASSERT_EQ(issuer.status(), absl::OkStatus()); + EXPECT_EQ(*issuer, expected_issuer_name_der); +} + +TEST_F(CrlUtils, IssuerFromCertNull) { + auto issuer = IssuerFromCert(nullptr); + EXPECT_EQ(issuer.status().code(), absl::StatusCode::kInvalidArgument); +} + } // namespace testing } // namespace grpc_core diff --git a/test/core/tsi/test_creds/crl_data/BUILD b/test/core/tsi/test_creds/crl_data/BUILD index ae3e58f7f72..d079cd5a7f0 100644 --- a/test/core/tsi/test_creds/crl_data/BUILD +++ b/test/core/tsi/test_creds/crl_data/BUILD @@ -25,4 +25,5 @@ exports_files([ "leaf_and_intermediate_chain.pem", "intermediate_ca.key", "intermediate_ca.pem", + "evil_ca.pem", ]) diff --git a/test/core/tsi/test_creds/crl_data/README b/test/core/tsi/test_creds/crl_data/README index 80f3568e95a..d19258c9440 100644 --- a/test/core/tsi/test_creds/crl_data/README +++ b/test/core/tsi/test_creds/crl_data/README @@ -46,6 +46,11 @@ Generate a chain with a leaf cert signed by an intermediate CA and revoke the in Run `intermediate_gen.sh` from the `test/core/tsi/test_creds/crl_data` directory +Generate a CA with the same issuer name but a different public key than the base CA +---------------------------------------------------------------------------- + +Run `evil_ca_gen.sh` from the `test/core/tsi/test_creds/crl_data` directory + Clean up: --------- diff --git a/test/core/tsi/test_creds/crl_data/crls/BUILD b/test/core/tsi/test_creds/crl_data/crls/BUILD index 922d279aa6a..ecbaa82a9df 100644 --- a/test/core/tsi/test_creds/crl_data/crls/BUILD +++ b/test/core/tsi/test_creds/crl_data/crls/BUILD @@ -19,4 +19,6 @@ exports_files([ "b9322cac.r0", "current.crl", "intermediate.crl", + "invalid_signature.crl", + "invalid_content.crl", ]) diff --git a/test/core/tsi/test_creds/crl_data/crls/invalid_content.crl b/test/core/tsi/test_creds/crl_data/crls/invalid_content.crl new file mode 100644 index 00000000000..22d8b48f0a0 --- /dev/null +++ b/test/core/tsi/test_creds/crl_data/crls/invalid_content.crl @@ -0,0 +1,15 @@ +-----BEGIN X509 CRL----- +AIICUDCCATgCAQEwDQYJKoZIhvcNAQELBQAwVjELMAkGA1UEBhMCQVUxEzARBgNV +BAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0 +ZDEPMA0GA1UEAwwGdGVzdGNhFw0yMzAzMDMxODA2NDNaFw0zMzAyMjgxODA2NDNa +MIGcMCUCFEpMyQOrk+uXDu20PhHwDJeua83mFw0yMzAzMDMxNjU5NTNaMCUCFEpM +yQOrk+uXDu20PhHwDJeua83nFw0yMzAzMDMxNzMxNDBaMCUCFEpMyQOrk+uXDu20 +PhHwDJeua83xFw0yMzAzMDMxODA2NDNaMCUCFFIgumScY9chZ0u8tUhjsOUh38hB +Fw0yMjAyMDQyMjExMTFaoA8wDTALBgNVHRQEBAICEAgwDQYJKoZIhvcNAQELBQAD +ggEBADohIZwm/gWLIc2yFJJbKzkdRmOq1s/MqnJxi5NutNumXTIPrZJqGzk8O4U6 +VasicIB2YD0o3arzUxCDyHv7VyJI7SVS0lqlmOxoOEOv2+CB6MxAOdKItkzbVVxu +0erx5HcKAGa7ZIAeekX1F1DcAgpN5Gt5uGhkMw3ObTCpEFRw+ZKET3WFQ6bG4AJ6 +GwOnNYG1LjaNigxG/k4K7A+grs/XnsNcpULbCROl7Qw4kyf1esrjS9utEO0YQQz4 +LgBTPZzQHlsirmxp+e5WR8LiDsKmbmAaBL+gV1Bkjj73c4pNJvoV/V1Ubdv0LCvH +DjrJtp10F0RGMRm6m9OuZYUSFzs= +-----END X509 CRL----- diff --git a/test/core/tsi/test_creds/crl_data/crls/invalid_signature.crl b/test/core/tsi/test_creds/crl_data/crls/invalid_signature.crl new file mode 100644 index 00000000000..2431949ce77 --- /dev/null +++ b/test/core/tsi/test_creds/crl_data/crls/invalid_signature.crl @@ -0,0 +1,15 @@ +-----BEGIN X509 CRL----- +MIICUDCCATgCAQEwDQYJKoZIhvcNAQELBQAwVjELMAkGA1UEBhMCQVUxEzARBgNV +BAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0 +ZDEPMA0GA1UEAwwGdGVzdGNhFw0yMzAzMDMxODA2NDNaFw0zMzAyMjgxODA2NDNa +MIGcMCUCFEpMyQOrk+uXDu20PhHwDJeua83mFw0yMzAzMDMxNjU5NTNaMCUCFEpM +yQOrk+uXDu20PhHwDJeua83nFw0yMzAzMDMxNzMxNDBaMCUCFEpMyQOrk+uXDu20 +PhHwDJeua83xFw0yMzAzMDMxODA2NDNaMCUCFFIgumScY9chZ0u8tUhjsOUh38hB +Fw0yMjAyMDQyMjExMTFaoA8wDTALBgNVHRQEBAICEAgwDQYJKoZIhvcNAQELBQAD +ggEBADohIZwm/gWLIc2yFJJbKzkdRmOq1s/MqnJxi5NutNumXTIPrZJqGzk8O4U6 +VasicIB2YD0o3arzUxCDyHv7VyJI7SVS0lqlmOxoOEOv2+CB6MxAOdKItkzbVVxu +0erx5HcKAGa7ZIAeekX1F1DcAgpN5Gt5uGhkMw3ObTCpEFRw+ZKET3WFQ6bG4AJ6 +GwOnNYG1LjaNigxG/k4K7A+grs/XnsNcpULbCROl7Qw4kyf1esrjS9utEO0YQQz4 +LgBTPZzQHlsirmxp+e5WR8LiDsKmbmAaBL+gV1Bkjj73c4pNJvoV/V1Ubdv0LCvH +DjrJtp10F0RGMRm6m9OuZYUSFza= +-----END X509 CRL----- diff --git a/test/core/tsi/test_creds/crl_data/evil_ca.key b/test/core/tsi/test_creds/crl_data/evil_ca.key new file mode 100644 index 00000000000..41614f98bc4 --- /dev/null +++ b/test/core/tsi/test_creds/crl_data/evil_ca.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC7RwHo8bWaioeI +oqq4qRkiRfqAl/XlaRCyygkMtkjuOy0LA42+LFXXNvDD8eVvVd3615Qopm0XzABd +iz2QiJBZH9qvvmZFg7vG4rbNMCHIN+0YYIOp5tJuyBUVhZ+/f/jZ+LoJeZgTRngQ +tMUmhs7kn4ttT+DC7ZHKhPf5vUokSPG4N2tBx21y2BzRup36q09vfvZeVEe5YxAM +KGWEOcCY/S5vTVeEJCqP2OfMmskIHq2cYWr6ZJzBpdhJXX6rTDWYlCzX49mzPrn6 +povhA/bENv9Gy1OHqPKt+EWEJCaurerkFwF74OG9zp/jCKZJTVkyxnCYjT2rYiDX +gWvNwdeHAgMBAAECggEADyya44Mzj0Y6jXV8tsIA0YLxCrAFZ7q3ydIj9z3ih+cP +PcK3yUPHYCJJUjR3PipWIP03Dy949xd7pMNjpXfjQPgbRz0lWpboxUiDvk7FlfcD +b4O2d12cCbI4Px+uHh1M48B1tnnTOtCYFDvJc6yITARUuZ03cs6UDwrvcB1dygsO +2sZLUOkWQb2DCMq86bxmkHvjuh3gj/CMTJv0Kprlo3YcKNgCwiNygEzlusyIcwpf +dU/SNoWcxY+F0F6wFC0uj75wWqDB6bmfCpY8Bb3Ey7TgWDTWjsB/NQsWbSxZ9o5i +qjQ6WSLKpLLLB/8dXxhk3Nz9tfonavBpLB+4fNpFFQKBgQDi61A3/U88iEo+sxMm +L3i0OS9g/mAnYQ7zYjq42eVyDTfa+eBck1Jmp1KEblfy7Eo3iyApNFoIzFz8va8N +tPNFK/K4mrf1aiFOk0SnvCstW8SBS99hBHXqrMnXrRh+L/OafM4sj88P4RbZxcIs +9RNiDIqcXAPDVU5aHIhs7CFzYwKBgQDTRyOR9PoTQnu0HV0ODDNzmP1eRWrXZ62N +khe9bm0TIG25Q1wsoR6MT5fxZlTe62FH7A5QgEheRtMctr+XGC2H+3N3MUxsTy37 +knPFiDl6Gs5DqKroewiDNbkziMOgctG/z6ORPiGghTRsn6y5dBaMstfvgip8fj5z +ytzgSfiujQKBgHZraOSfK++iDGTmHRMraOlcgm4ysck7LIs08wIurD+1yDVde4m0 +VCdAIJ792qXqS9zqnPED4gx/YfN/pdAYY2/wvG08SM4pAZK45fZHC51TK5xyFPPT +WRoL7BXCvmpz6cPwZ8P3lI5r3/nr6yZ9Cw17EAcDOe+BIC+EfmmhXN+TAoGBAIp0 +oDbSV9+vPen3JDhEfqNOqxvQWgf3haC1EKGvcAOMyNsT7Z/BpodE0cn8ybmcfw/m +/ip7JvHBcC/tAvk9evkWK8D8qZyA9x1aCEx2zVPbpThpnDbmCdoSpt/CzJClLheJ +NyPDl73eDVDyAvs1vGFQAnqOztDu2nZ/huflEfcxAoGAbLUQV5PjqJrsIosEMXsv +qOzQZ5BBEk/jo9zqYSNXWVs0I9Invj5iAYewoM5qn9DFQ3q3O/mPHxF6HT7JHfjn +T8wdOTQk5L1yaaSFsiti3C3AQ2zShT1k6m3V+mf0iWJw878LCURQQFNIHu7zVdXy +4xwQpVw2CN7iufRYN7kOcDo= +-----END PRIVATE KEY----- diff --git a/test/core/tsi/test_creds/crl_data/evil_ca.pem b/test/core/tsi/test_creds/crl_data/evil_ca.pem new file mode 100644 index 00000000000..6708690b1db --- /dev/null +++ b/test/core/tsi/test_creds/crl_data/evil_ca.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDeTCCAmGgAwIBAgIUULA9nt1NB3W1i4RevrKeRQQLkaIwDQYJKoZIhvcNAQEL +BQAwVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGdGVzdGNhMB4XDTI0 +MDEyMjIxNDAyMFoXDTM0MDExOTIxNDAyMFowVjELMAkGA1UEBhMCQVUxEzARBgNV +BAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0 +ZDEPMA0GA1UEAwwGdGVzdGNhMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC +AQEAu0cB6PG1moqHiKKquKkZIkX6gJf15WkQssoJDLZI7jstCwONvixV1zbww/Hl +b1Xd+teUKKZtF8wAXYs9kIiQWR/ar75mRYO7xuK2zTAhyDftGGCDqebSbsgVFYWf +v3/42fi6CXmYE0Z4ELTFJobO5J+LbU/gwu2RyoT3+b1KJEjxuDdrQcdtctgc0bqd ++qtPb372XlRHuWMQDChlhDnAmP0ub01XhCQqj9jnzJrJCB6tnGFq+mScwaXYSV1+ +q0w1mJQs1+PZsz65+qaL4QP2xDb/RstTh6jyrfhFhCQmrq3q5BcBe+Dhvc6f4wim +SU1ZMsZwmI09q2Ig14FrzcHXhwIDAQABoz8wPTAMBgNVHRMEBTADAQH/MA4GA1Ud +DwEB/wQEAwIBBjAdBgNVHQ4EFgQUjcQvfJ6kAUgljgToPpQ0DmCW0Q8wDQYJKoZI +hvcNAQELBQADggEBALLNhOYqlhOcCsTD1SPfm9MAjfpV1EjSjDCpIfwCk5gI2CUX +g7MyUzn2gQJUiYx74BKmjv6W/sLzNxqR0wZQUr4d/7HX+Lm0xCCYdIUELEM8lZ30 +maBJ599cQnLXDB1ZFEekj3DMM6jL7OQnBaDs5jW4GcDcuwd5cgXfgIaZVjBVJ11Y +CFAhIuh5CM8xhqxWYWY+h0VLU64s8WCNrBEy1OU5KpQRfpd4cvpoWn7E1SfhK1Iq +Bp+1k4oDBpGGw4NLXI3i1aU8x1+KoXxNRg5dOED0OLgppvaWB2yIpqBlcZDaNpq4 +P+WFGBiSUpWU5yYwCDvQAgTWtWkmyflVwslHaGs= +-----END CERTIFICATE----- diff --git a/test/core/tsi/test_creds/crl_data/evil_ca_gen.sh b/test/core/tsi/test_creds/crl_data/evil_ca_gen.sh new file mode 100644 index 00000000000..31dec70057c --- /dev/null +++ b/test/core/tsi/test_creds/crl_data/evil_ca_gen.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Copyright 2024 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generates a CA with the same issuer name as the good CA in this directory +openssl req -x509 -new -newkey rsa:2048 -nodes -keyout evil_ca.key -out evil_ca.pem \ + -config ca-openssl.cnf -days 3650 -extensions v3_req diff --git a/test/core/tsi/transport_security_test_lib.cc b/test/core/tsi/transport_security_test_lib.cc index 811ecff0076..a0be6edf466 100644 --- a/test/core/tsi/transport_security_test_lib.cc +++ b/test/core/tsi/transport_security_test_lib.cc @@ -762,3 +762,27 @@ std::string GenerateSelfSignedCertificate( BN_free(n); return pem; } + +X509* ReadPemCert(absl::string_view pem_cert) { + BIO* cert_bio = + BIO_new_mem_buf(pem_cert.data(), static_cast(pem_cert.size())); + // Errors on BIO + if (cert_bio == nullptr) { + return nullptr; + } + X509* cert = PEM_read_bio_X509(cert_bio, nullptr, nullptr, nullptr); + BIO_free(cert_bio); + return cert; +} + +X509_CRL* ReadCrl(absl::string_view crl_pem) { + BIO* crl_bio = + BIO_new_mem_buf(crl_pem.data(), static_cast(crl_pem.size())); + // Errors on BIO + if (crl_bio == nullptr) { + return nullptr; + } + X509_CRL* crl = PEM_read_bio_X509_CRL(crl_bio, nullptr, nullptr, nullptr); + BIO_free(crl_bio); + return crl; +} diff --git a/test/core/tsi/transport_security_test_lib.h b/test/core/tsi/transport_security_test_lib.h index 69f07c1269c..6e3efec2bff 100644 --- a/test/core/tsi/transport_security_test_lib.h +++ b/test/core/tsi/transport_security_test_lib.h @@ -19,6 +19,8 @@ #ifndef GRPC_TEST_CORE_TSI_TRANSPORT_SECURITY_TEST_LIB_H #define GRPC_TEST_CORE_TSI_TRANSPORT_SECURITY_TEST_LIB_H +#include + #include #include "src/core/tsi/transport_security_interface.h" @@ -238,4 +240,10 @@ struct SelfSignedCertificateOptions { std::string GenerateSelfSignedCertificate( const SelfSignedCertificateOptions& options); +// Returns the OpenSSL representation of a PEM cert. +X509* ReadPemCert(absl::string_view pem_cert); + +// Returns the OpenSSL representation of a CRL. +X509_CRL* ReadCrl(absl::string_view crl_pem); + #endif // GRPC_TEST_CORE_TSI_TRANSPORT_SECURITY_TEST_LIB_H From 6f406a7a68329bb2065ad28b16111a7048548232 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Fri, 16 Feb 2024 10:13:56 -0800 Subject: [PATCH 26/37] Updated libprotobuf_mutator to the latest (#35833) This is partially related to protobuf upgrade (https://github.com/grpc/grpc/pull/35796) but it's not strictly required. But it's still a good thing to get this up-to-dated. Closes #35833 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/35833 from veblush:libprotobuf_mutator_update 0fcc0c19dc1c9d973cb87ee80f1427a6f9b15c28 PiperOrigin-RevId: 607726401 --- bazel/grpc_deps.bzl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bazel/grpc_deps.bzl b/bazel/grpc_deps.bzl index bd3763579e1..6334e144d5f 100644 --- a/bazel/grpc_deps.bzl +++ b/bazel/grpc_deps.bzl @@ -618,11 +618,11 @@ def grpc_test_only_deps(): if "com_google_libprotobuf_mutator" not in native.existing_rules(): http_archive( name = "com_google_libprotobuf_mutator", - sha256 = "11ab4c57b4051977d8fedb86dba5c9092e578bc293c47be146e0b0596b6a0bdc", + sha256 = "9c8f800aed088cdf89adc3eaaa66b56b4da7da041f26338aa71a2ab43d860d46", urls = [ - "https://storage.googleapis.com/grpc-bazel-mirror/github.com/google/libprotobuf-mutator/archive/c390388561be36f94a559a4aed7e2fe60470f60b.tar.gz", - "https://github.com/google/libprotobuf-mutator/archive/c390388561be36f94a559a4aed7e2fe60470f60b.tar.gz", + "https://storage.googleapis.com/grpc-bazel-mirror/github.com/google/libprotobuf-mutator/archive/1f95f8083066f5b38fd2db172e7e7f9aa7c49d2d.tar.gz", + "https://github.com/google/libprotobuf-mutator/archive/1f95f8083066f5b38fd2db172e7e7f9aa7c49d2d.tar.gz", ], - strip_prefix = "libprotobuf-mutator-c390388561be36f94a559a4aed7e2fe60470f60b", + strip_prefix = "libprotobuf-mutator-1f95f8083066f5b38fd2db172e7e7f9aa7c49d2d", build_file = "@com_github_grpc_grpc//third_party:libprotobuf_mutator.BUILD", ) From b85542335968dd9ab8974cd928bbed819dfe873c Mon Sep 17 00:00:00 2001 From: apolcyn Date: Fri, 16 Feb 2024 10:19:37 -0800 Subject: [PATCH 27/37] [testing] Add StopListening helper and BidiStream handler to XDS e2e test lib (#35861) As title. Pulling these additions out from a larger change. Related: cl/563857636 Closes #35861 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/35861 from apolcyn:xds_test_changes a67f64d93c8d1edf38ea17b98a8725d536c4e104 PiperOrigin-RevId: 607728140 --- test/cpp/end2end/xds/xds_end2end_test_lib.cc | 10 ++++++++++ test/cpp/end2end/xds/xds_end2end_test_lib.h | 2 ++ 2 files changed, 12 insertions(+) diff --git a/test/cpp/end2end/xds/xds_end2end_test_lib.cc b/test/cpp/end2end/xds/xds_end2end_test_lib.cc index e91e4af4511..b95d91a1dd9 100644 --- a/test/cpp/end2end/xds/xds_end2end_test_lib.cc +++ b/test/cpp/end2end/xds/xds_end2end_test_lib.cc @@ -171,6 +171,16 @@ void XdsEnd2endTest::ServerThread::StopListeningAndSendGoaways() { gpr_log(GPR_INFO, "%s done sending GOAWAYs", Type()); } +void XdsEnd2endTest::ServerThread::StopListening() { + gpr_log(GPR_INFO, "%s about to stop listening", Type()); + { + grpc_core::ExecCtx exec_ctx; + auto* server = grpc_core::Server::FromC(server_->c_server()); + server->StopListening(); + } + gpr_log(GPR_INFO, "%s stopped listening", Type()); +} + void XdsEnd2endTest::ServerThread::Serve(grpc_core::Mutex* mu, grpc_core::CondVar* cond) { // We need to acquire the lock here in order to prevent the notify_one diff --git a/test/cpp/end2end/xds/xds_end2end_test_lib.h b/test/cpp/end2end/xds/xds_end2end_test_lib.h index cdb98d90591..1b9ade11905 100644 --- a/test/cpp/end2end/xds/xds_end2end_test_lib.h +++ b/test/cpp/end2end/xds/xds_end2end_test_lib.h @@ -259,6 +259,8 @@ class XdsEnd2endTest : public ::testing::TestWithParam, allow_put_requests_ = allow_put_requests; } + void StopListening(); + void StopListeningAndSendGoaways(); private: From ee3153b20d6f9ab3aa0d394a291180bc511a266d Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Fri, 16 Feb 2024 11:18:09 -0800 Subject: [PATCH 28/37] [experiments] Extend deadline for keepalive_server_fix (#35928) Closes #35928 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/35928 from yashykt:IncreaseExperimentDeadline da53c605478683cf307a8058083b7a57dc64292f PiperOrigin-RevId: 607746309 --- src/core/lib/experiments/experiments.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/experiments/experiments.yaml b/src/core/lib/experiments/experiments.yaml index 87eeaa32f13..d00ec7e1986 100644 --- a/src/core/lib/experiments/experiments.yaml +++ b/src/core/lib/experiments/experiments.yaml @@ -126,7 +126,7 @@ description: Allows overriding keepalive_permit_without_calls for servers. Refer https://github.com/grpc/grpc/pull/33917 for more information. - expiry: 2023/12/31 + expiry: 2024/12/31 owner: yashkt@google.com test_tags: [] allow_in_fuzzing_config: false From e3b5ec6e1726187e56612eaad7ff8d612163ab24 Mon Sep 17 00:00:00 2001 From: Thomas Braun Date: Fri, 16 Feb 2024 11:36:49 -0800 Subject: [PATCH 29/37] ssl_transport_security.cc: Make it compile with libressl (#35615) Compiling grpc latest master [1] on Windows 10 Pro using MSVC 2019 and using LibreSSL 3.8.2 as SSL provider (package) does currently not work, as the functions X509_STORE_set_get_crl/X509_STORE_set_check_crl and the define SSL_OP_NO_RENEGOTIATION are not present. Employ a workaround seen in the surrounding code by making the code block only for openssl available. [1]: 24f89637 ([Test] Removed obsolete node test from run_tests.py (#35525), 2024-01-19) Closes #35615 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/35615 from t-b:fix-compiliation-libressl 858f430715fa54fc010b715ae917120e50983fe5 PiperOrigin-RevId: 607752281 --- src/core/tsi/ssl_transport_security.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/core/tsi/ssl_transport_security.cc b/src/core/tsi/ssl_transport_security.cc index ef79db1c0a2..4d8ac159d06 100644 --- a/src/core/tsi/ssl_transport_security.cc +++ b/src/core/tsi/ssl_transport_security.cc @@ -2081,7 +2081,7 @@ tsi_result tsi_create_ssl_client_handshaker_factory_with_options( #else ssl_context = SSL_CTX_new(TLSv1_2_method()); #endif -#if OPENSSL_VERSION_NUMBER >= 0x10101000 +#if OPENSSL_VERSION_NUMBER >= 0x10101000 && !defined(LIBRESSL_VERSION_NUMBER) SSL_CTX_set_options(ssl_context, SSL_OP_NO_RENEGOTIATION); #endif if (ssl_context == nullptr) { @@ -2184,7 +2184,7 @@ tsi_result tsi_create_ssl_client_handshaker_factory_with_options( nullptr); } -#if OPENSSL_VERSION_NUMBER >= 0x10100000 +#if OPENSSL_VERSION_NUMBER >= 0x10100000 && !defined(LIBRESSL_VERSION_NUMBER) if (options->crl_provider != nullptr) { SSL_CTX_set_ex_data(impl->ssl_context, g_ssl_ctx_ex_crl_provider_index, options->crl_provider.get()); @@ -2301,7 +2301,7 @@ tsi_result tsi_create_ssl_server_handshaker_factory_with_options( #else impl->ssl_contexts[i] = SSL_CTX_new(TLSv1_2_method()); #endif -#if OPENSSL_VERSION_NUMBER >= 0x10101000 +#if OPENSSL_VERSION_NUMBER >= 0x10101000 && !defined(LIBRESSL_VERSION_NUMBER) SSL_CTX_set_options(impl->ssl_contexts[i], SSL_OP_NO_RENEGOTIATION); #endif if (impl->ssl_contexts[i] == nullptr) { @@ -2388,7 +2388,7 @@ tsi_result tsi_create_ssl_server_handshaker_factory_with_options( break; } -#if OPENSSL_VERSION_NUMBER >= 0x10100000 +#if OPENSSL_VERSION_NUMBER >= 0x10100000 && !defined(LIBRESSL_VERSION_NUMBER) if (options->crl_provider != nullptr) { SSL_CTX_set_ex_data(impl->ssl_contexts[i], g_ssl_ctx_ex_crl_provider_index, From 8d094cdba99c523d4ecb813cbc2db1ac297cc650 Mon Sep 17 00:00:00 2001 From: Yousuk Seung Date: Fri, 16 Feb 2024 15:47:22 -0800 Subject: [PATCH 30/37] [tracing] Add target_write_size to HttpAnnotation (#35921) Closes #35921 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/35921 from yousukseung:wire-size-target 6c01052721856ca6cc7db88d9f17d33786a7801b PiperOrigin-RevId: 607822267 --- .../chttp2/transport/chttp2_transport.cc | 17 +++++------- .../chttp2/transport/chttp2_transport.h | 26 ++++++++++++++++--- .../ext/transport/chttp2/transport/writing.cc | 21 ++++++++++----- 3 files changed, 43 insertions(+), 21 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc index f917662bf8e..c30dde9de1a 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc @@ -338,16 +338,11 @@ void ForEachContextListEntryExecute(void* arg, Timestamps* ts, delete context_list; } -HttpAnnotation::HttpAnnotation( - Type type, gpr_timespec time, - absl::optional transport_stats, - absl::optional stream_stats) +HttpAnnotation::HttpAnnotation(Type type, gpr_timespec time) : CallTracerAnnotationInterface::Annotation( CallTracerAnnotationInterface::AnnotationType::kHttpTransport), type_(type), - time_(time), - transport_stats_(transport_stats), - stream_stats_(stream_stats) {} + time_(time) {} std::string HttpAnnotation::ToString() const { std::string s = "HttpAnnotation type: "; @@ -1433,9 +1428,11 @@ static void perform_stream_op_locked(void* stream_op, if (op->send_initial_metadata) { if (s->call_tracer) { - s->call_tracer->RecordAnnotation(grpc_core::HttpAnnotation( - grpc_core::HttpAnnotation::Type::kStart, gpr_now(GPR_CLOCK_REALTIME), - s->t->flow_control.stats(), s->flow_control.stats())); + s->call_tracer->RecordAnnotation( + grpc_core::HttpAnnotation(grpc_core::HttpAnnotation::Type::kStart, + gpr_now(GPR_CLOCK_REALTIME)) + .Add(s->t->flow_control.stats()) + .Add(s->flow_control.stats())); } if (t->is_client && t->channelz_socket != nullptr) { t->channelz_socket->RecordStreamStartedFromLocal(); diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.h b/src/core/ext/transport/chttp2/transport/chttp2_transport.h index ddf7539606a..477137ea624 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.h +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.h @@ -112,10 +112,27 @@ class HttpAnnotation : public CallTracerAnnotationInterface::Annotation { kEnd, }; - HttpAnnotation( - Type type, gpr_timespec time, - absl::optional transport_stats, - absl::optional stream_stats); + // A snapshot of write stats to export. + struct WriteStats { + size_t target_write_size; + }; + + HttpAnnotation(Type type, gpr_timespec time); + + HttpAnnotation& Add(const chttp2::TransportFlowControl::Stats& stats) { + transport_stats_ = stats; + return *this; + } + + HttpAnnotation& Add(const chttp2::StreamFlowControl::Stats& stats) { + stream_stats_ = stats; + return *this; + } + + HttpAnnotation& Add(const WriteStats& stats) { + write_stats_ = stats; + return *this; + } std::string ToString() const override; @@ -133,6 +150,7 @@ class HttpAnnotation : public CallTracerAnnotationInterface::Annotation { const gpr_timespec time_; absl::optional transport_stats_; absl::optional stream_stats_; + absl::optional write_stats_; }; } // namespace grpc_core diff --git a/src/core/ext/transport/chttp2/transport/writing.cc b/src/core/ext/transport/chttp2/transport/writing.cc index b1f9f30d2fc..4955d041bcc 100644 --- a/src/core/ext/transport/chttp2/transport/writing.cc +++ b/src/core/ext/transport/chttp2/transport/writing.cc @@ -492,10 +492,15 @@ class StreamWriteContext { t_, s_, &s_->send_initial_metadata_finished, absl::OkStatus(), "send_initial_metadata_finished"); if (s_->call_tracer) { - s_->call_tracer->RecordAnnotation(grpc_core::HttpAnnotation( - grpc_core::HttpAnnotation::Type::kHeadWritten, - gpr_now(GPR_CLOCK_REALTIME), s_->t->flow_control.stats(), - s_->flow_control.stats())); + grpc_core::HttpAnnotation::WriteStats write_stats; + write_stats.target_write_size = write_context_->target_write_size(); + s_->call_tracer->RecordAnnotation( + grpc_core::HttpAnnotation( + grpc_core::HttpAnnotation::Type::kHeadWritten, + gpr_now(GPR_CLOCK_REALTIME)) + .Add(s_->t->flow_control.stats()) + .Add(s_->flow_control.stats()) + .Add(write_stats)); } } @@ -619,9 +624,11 @@ class StreamWriteContext { grpc_chttp2_mark_stream_closed(t_, s_, !t_->is_client, true, absl::OkStatus()); if (s_->call_tracer) { - s_->call_tracer->RecordAnnotation(grpc_core::HttpAnnotation( - grpc_core::HttpAnnotation::Type::kEnd, gpr_now(GPR_CLOCK_REALTIME), - s_->t->flow_control.stats(), s_->flow_control.stats())); + s_->call_tracer->RecordAnnotation( + grpc_core::HttpAnnotation(grpc_core::HttpAnnotation::Type::kEnd, + gpr_now(GPR_CLOCK_REALTIME)) + .Add(s_->t->flow_control.stats()) + .Add(s_->flow_control.stats())); } } From 7a43130d106449d892737a1b2b991509d8f0dcf3 Mon Sep 17 00:00:00 2001 From: apolcyn Date: Fri, 16 Feb 2024 16:43:13 -0800 Subject: [PATCH 31/37] [ruby] fix ruby interop build (#35934) Followup fix for #35911 Closes #35934 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/35934 from apolcyn:ruby_stuff fdafeffc05e1c2f5da248a78054aeca6f1813894 PiperOrigin-RevId: 607836125 --- tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh index 703bdd92ceb..2558e71cddd 100755 --- a/tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh +++ b/tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh @@ -27,7 +27,7 @@ ${name}') cp -r /var/local/jenkins/service_account $HOME || true cd /var/local/git/grpc -rvm --default use ruby-2.7 +rvm --default use ruby-3.0 # build Ruby interop client and server (cd src/ruby && gem install bundler -v 2.4.22 && bundle && bundle exec rake compile) From 88c5f25997aeda104ac3ec84291616f4e606c030 Mon Sep 17 00:00:00 2001 From: Xuan Wang Date: Fri, 16 Feb 2024 16:52:29 -0800 Subject: [PATCH 32/37] [Python Otel] Fix packaging issue (#35929) --- src/python/grpcio_observability/README.rst | 5 +++-- src/python/grpcio_observability/setup.py | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/python/grpcio_observability/README.rst b/src/python/grpcio_observability/README.rst index a4122f79bd8..0e5df12528b 100644 --- a/src/python/grpcio_observability/README.rst +++ b/src/python/grpcio_observability/README.rst @@ -1,11 +1,12 @@ gRPC Python Observability -=========== +========================= + Package for gRPC Python Observability. More details can be found in `OpenTelemetry Metrics gRFC `_. How gRPC Python Observability Works -------------------------- +----------------------------------- gRPC Python is a wrapper layer built upon the gRPC Core (written in C/C++). Most of telemetry data is collected at core layer and then exported to Python layer. To optimize performance and reduce diff --git a/src/python/grpcio_observability/setup.py b/src/python/grpcio_observability/setup.py index f9bd1966504..3c28ed7ae9a 100644 --- a/src/python/grpcio_observability/setup.py +++ b/src/python/grpcio_observability/setup.py @@ -40,7 +40,7 @@ import grpc_version _parallel_compile_patch.monkeypatch_compile_maybe() CLASSIFIERS = [ - "Private :: Do Not Upload", + "Development Status :: 4 - Beta", "Programming Language :: Python", "Programming Language :: Python :: 3", "License :: OSI Approved :: Apache Software License", @@ -257,6 +257,7 @@ setuptools.setup( name="grpcio-observability", version=grpc_version.VERSION, description="gRPC Python observability package", + long_description_content_type="text/x-rst", long_description=open(README_PATH, "r").read(), author="The gRPC Authors", author_email="grpc-io@googlegroups.com", From 5227db884d77d575158adc3bcafed6a423c5b8c4 Mon Sep 17 00:00:00 2001 From: apolcyn Date: Fri, 16 Feb 2024 17:22:47 -0800 Subject: [PATCH 33/37] [pick first] explicitly unset selected subchannel when promoting pending subchannel (#35865) This change is needed in order to get certain LB-policy-unit-test-framework-based tests to pass, which involve updates on PF policies. Without this change, in such tests, health watchers on the original subchannel can be left hanging around. The underlying reason is related to https://github.com/grpc/grpc/blob/842057d8d5a79d48538e4b276f46d56b4dfcdb16/test/core/client_channel/lb_policy/lb_policy_test_lib.h#L203. Related: cl/563857636 Closes #35865 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/35865 from apolcyn:update_pf_Test 1a5e6b5591e9e210d74b533d08950271b4518107 PiperOrigin-RevId: 607844309 --- .../load_balancing/pick_first/pick_first.cc | 1 + .../lb_policy/lb_policy_test_lib.h | 9 ++-- .../lb_policy/pick_first_test.cc | 48 ++++++++++++++++++- 3 files changed, 54 insertions(+), 4 deletions(-) diff --git a/src/core/load_balancing/pick_first/pick_first.cc b/src/core/load_balancing/pick_first/pick_first.cc index 9d3733ddbed..406f388ed34 100644 --- a/src/core/load_balancing/pick_first/pick_first.cc +++ b/src/core/load_balancing/pick_first/pick_first.cc @@ -1022,6 +1022,7 @@ void PickFirst::SubchannelList::SubchannelData::ProcessUnselectedReadyLocked() { p, p->latest_pending_subchannel_list_.get(), p->subchannel_list_.get()); } + p->UnsetSelectedSubchannel(); p->subchannel_list_ = std::move(p->latest_pending_subchannel_list_); } // Cases 1 and 2. diff --git a/test/core/client_channel/lb_policy/lb_policy_test_lib.h b/test/core/client_channel/lb_policy/lb_policy_test_lib.h index 3b75537461e..e7c49e794d0 100644 --- a/test/core/client_channel/lb_policy/lb_policy_test_lib.h +++ b/test/core/client_channel/lb_policy/lb_policy_test_lib.h @@ -698,8 +698,10 @@ class LoadBalancingPolicyTest : public ::testing::Test { const absl::optional backend_metric_data_; }; - explicit LoadBalancingPolicyTest(absl::string_view lb_policy_name) - : lb_policy_name_(lb_policy_name) {} + explicit LoadBalancingPolicyTest(absl::string_view lb_policy_name, + ChannelArgs channel_args = ChannelArgs()) + : lb_policy_name_(lb_policy_name), + channel_args_(std::move(channel_args)) {} void SetUp() override { // Order is important here: Fuzzing EE needs to be created before @@ -715,7 +717,7 @@ class LoadBalancingPolicyTest : public ::testing::Test { auto helper = std::make_unique(this); helper_ = helper.get(); LoadBalancingPolicy::Args args = {work_serializer_, std::move(helper), - ChannelArgs()}; + channel_args_}; lb_policy_ = CoreConfiguration::Get().lb_policy_registry().CreateLoadBalancingPolicy( lb_policy_name_, std::move(args)); @@ -1491,6 +1493,7 @@ class LoadBalancingPolicyTest : public ::testing::Test { std::map subchannel_pool_; OrphanablePtr lb_policy_; const absl::string_view lb_policy_name_; + const ChannelArgs channel_args_; }; } // namespace testing diff --git a/test/core/client_channel/lb_policy/pick_first_test.cc b/test/core/client_channel/lb_policy/pick_first_test.cc index fccc8af16ee..77ba738d1a0 100644 --- a/test/core/client_channel/lb_policy/pick_first_test.cc +++ b/test/core/client_channel/lb_policy/pick_first_test.cc @@ -14,6 +14,8 @@ // limitations under the License. // +#include "src/core/load_balancing/pick_first/pick_first.h" + #include #include @@ -54,7 +56,8 @@ namespace { class PickFirstTest : public LoadBalancingPolicyTest { protected: - PickFirstTest() : LoadBalancingPolicyTest("pick_first") {} + explicit PickFirstTest(ChannelArgs channel_args = ChannelArgs()) + : LoadBalancingPolicyTest("pick_first", channel_args) {} void SetUp() override { LoadBalancingPolicyTest::SetUp(); @@ -1103,6 +1106,49 @@ TEST_F(PickFirstTest, ShufflingDisabled) { EXPECT_THAT(address_order, ::testing::ElementsAreArray(kAddresses)); } } + +class PickFirstHealthCheckingEnabledTest : public PickFirstTest { + protected: + PickFirstHealthCheckingEnabledTest() + : PickFirstTest(ChannelArgs().Set( + GRPC_ARG_INTERNAL_PICK_FIRST_ENABLE_HEALTH_CHECKING, true)) {} +}; + +TEST_F(PickFirstHealthCheckingEnabledTest, UpdateWithReadyChannel) { + constexpr absl::string_view kAddress = "ipv4:127.0.0.1:443"; + LoadBalancingPolicy::UpdateArgs update = + BuildUpdate({kAddress}, MakePickFirstConfig()); + absl::Status status = ApplyUpdate(update, lb_policy()); + EXPECT_TRUE(status.ok()) << status; + // LB policy should have created a subchannel for the address. + auto* subchannel = FindSubchannel(kAddress); + ASSERT_NE(subchannel, nullptr); + // When the LB policy receives the first subchannel's initial connectivity + // state notification (IDLE), it will request a connection. + EXPECT_TRUE(subchannel->ConnectionRequested()); + // This causes the subchannel to start to connect, so it reports CONNECTING. + subchannel->SetConnectivityState(GRPC_CHANNEL_CONNECTING); + // LB policy should have reported CONNECTING state. + ExpectConnectingUpdate(); + // When the subchannel becomes connected, it reports READY. + subchannel->SetConnectivityState(GRPC_CHANNEL_READY); + // The LB policy will report CONNECTING some number of times (doesn't + // matter how many) and then report READY. + auto picker = WaitForConnected(); + ASSERT_NE(picker, nullptr); + EXPECT_EQ(ExpectPickComplete(picker.get()), kAddress); + // Reapply the same update we did before. The the underlying + // subchannel will immediately become ready. + status = + ApplyUpdate(BuildUpdate({kAddress}, MakePickFirstConfig()), lb_policy()); + EXPECT_TRUE(status.ok()) << status; + picker = ExpectState(GRPC_CHANNEL_READY); + EXPECT_EQ(ExpectPickComplete(picker.get()), kAddress); + // At this point, NumWatchers() should account for our + // subchannel connectivity watcher and our health watcher. + EXPECT_EQ(subchannel->NumWatchers(), 2); +} + } // namespace } // namespace testing } // namespace grpc_core From 469449d95ea28ad70961d062617ee56a052f289f Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 20 Feb 2024 09:51:26 -0800 Subject: [PATCH 34/37] [chaotic-good] Connect timeout is in seconds, not nanoseconds (#35938) Unit conversion was a bit off here :) Closes #35938 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/35938 from ctiller:timeout43 e5eacb44782f2a20816a2df253d7150e5134f690 PiperOrigin-RevId: 608641323 --- .../transport/chaotic_good/client/chaotic_good_connector.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc b/src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc index 6be26f47c43..856e3e56740 100644 --- a/src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc +++ b/src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc @@ -139,7 +139,7 @@ auto ChaoticGoodConnector::WaitForDataEndpointSetup( self->args_.channel_args), ResourceQuota::Default()->memory_quota()->CreateMemoryAllocator( "data_endpoint_connection"), - EventEngine::Duration(kTimeoutSecs)); + std::chrono::seconds(kTimeoutSecs)); return TrySeq(Race( TrySeq(self->data_endpoint_ready_.Wait(), @@ -251,7 +251,7 @@ void ChaoticGoodConnector::Connect(const Args& args, Result* result, args_.channel_args), ResourceQuota::Default()->memory_quota()->CreateMemoryAllocator( "data_endpoint_connection"), - EventEngine::Duration(kTimeoutSecs)); + std::chrono::seconds(kTimeoutSecs)); } void ChaoticGoodConnector::OnHandshakeDone(void* arg, grpc_error_handle error) { From 179e0f952f48be9eb4e7303fd7fbb794a9855377 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Tue, 20 Feb 2024 11:34:15 -0800 Subject: [PATCH 35/37] [build] move parts of the `grpc_base` BUILD target out into their own targets (#35846) This breaks the following pieces out of the `grpc_base` BUILD target: - channelz - call_trace - dynamic_annotations - call_combiner - resource_quota_api - iomgr More work is still needed to pull apart the remaining parts of `grpc_base`. Closes #35846 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/35846 from markdroth:client_channel_build_dep_cycle_fix f1a9b6c2b2e5aa230ac5434655fc49b9282d81fd PiperOrigin-RevId: 608680098 --- BUILD | 351 +++++++++++++++--- CMakeLists.txt | 3 - Makefile | 2 - Package.swift | 1 - build_autogenerated.yaml | 3 - config.m4 | 1 - config.w32 | 1 - gRPC-Core.podspec | 1 - grpc.gemspec | 1 - grpc.gyp | 3 - package.xml | 1 - src/core/BUILD | 34 ++ src/core/lib/channel/call_tracer.h | 1 - src/core/lib/channel/channel_stack.cc | 7 + .../lib/channel/channel_stack_builder_impl.cc | 140 +++++++ src/core/lib/surface/call_trace.cc | 163 -------- src/core/lib/surface/call_trace.h | 6 - src/core/lib/transport/transport_op_string.cc | 8 - src/cpp/ext/gcp/BUILD | 2 + src/python/grpcio/grpc_core_dependencies.py | 1 - tools/doxygen/Doxyfile.c++.internal | 1 - tools/doxygen/Doxyfile.core.internal | 1 - 22 files changed, 477 insertions(+), 255 deletions(-) delete mode 100644 src/core/lib/surface/call_trace.cc diff --git a/BUILD b/BUILD index ca76d4c83b4..5dd9ed6b08d 100644 --- a/BUILD +++ b/BUILD @@ -1284,21 +1284,123 @@ grpc_cc_library( ) grpc_cc_library( - name = "grpc_base", + name = "channelz", srcs = [ - "//src/core:lib/channel/call_tracer.cc", - "//src/core:lib/channel/channel_stack.cc", - "//src/core:lib/channel/channel_stack_builder_impl.cc", "//src/core:lib/channel/channel_trace.cc", "//src/core:lib/channel/channelz.cc", "//src/core:lib/channel/channelz_registry.cc", - "//src/core:lib/channel/connected_channel.cc", - "//src/core:lib/channel/promise_based_filter.cc", - "//src/core:lib/channel/server_call_tracer_filter.cc", - "//src/core:lib/channel/status_util.cc", - "//src/core:lib/compression/compression.cc", - "//src/core:lib/compression/message_compress.cc", + ], + hdrs = [ + "//src/core:lib/channel/channel_trace.h", + "//src/core:lib/channel/channelz.h", + "//src/core:lib/channel/channelz_registry.h", + ], + external_deps = [ + "absl/base:core_headers", + "absl/status:statusor", + "absl/strings", + "absl/types:optional", + ], + language = "c++", + deps = [ + "exec_ctx", + "gpr", + "grpc_public_hdrs", + "parse_address", + "ref_counted_ptr", + "sockaddr_utils", + "uri_parser", + "//src/core:channel_args", + "//src/core:connectivity_state", + "//src/core:json", + "//src/core:json_writer", + "//src/core:per_cpu", + "//src/core:ref_counted", + "//src/core:resolved_address", + "//src/core:slice", + "//src/core:time", + "//src/core:useful", + ], +) + +grpc_cc_library( + name = "call_trace", + hdrs = [ + "//src/core:lib/surface/call_trace.h", + ], + language = "c++", + deps = [ + "grpc_trace", + ], +) + +grpc_cc_library( + name = "dynamic_annotations", + hdrs = [ + "//src/core:lib/iomgr/dynamic_annotations.h", + ], + language = "c++", + deps = [ + "gpr_public_hdrs", + ], +) + +grpc_cc_library( + name = "call_combiner", + srcs = [ "//src/core:lib/iomgr/call_combiner.cc", + ], + hdrs = [ + "//src/core:lib/iomgr/call_combiner.h", + ], + external_deps = [ + "absl/container:inlined_vector", + ], + language = "c++", + deps = [ + "dynamic_annotations", + "exec_ctx", + "gpr", + "ref_counted_ptr", + "stats", + "//src/core:closure", + "//src/core:gpr_atm", + "//src/core:ref_counted", + "//src/core:stats_data", + ], +) + +grpc_cc_library( + name = "resource_quota_api", + srcs = [ + "//src/core:lib/resource_quota/api.cc", + ], + hdrs = [ + "//src/core:lib/resource_quota/api.h", + ], + external_deps = [ + "absl/strings", + ], + language = "c++", + visibility = ["@grpc:alt_grpc_base_legacy"], + deps = [ + "channel_arg_names", + "config", + "event_engine_base_hdrs", + "exec_ctx", + "gpr_public_hdrs", + "grpc_public_hdrs", + "ref_counted_ptr", + "//src/core:channel_args", + "//src/core:memory_quota", + "//src/core:resource_quota", + "//src/core:thread_quota", + ], +) + +grpc_cc_library( + name = "iomgr", + srcs = [ "//src/core:lib/iomgr/cfstream_handle.cc", "//src/core:lib/iomgr/dualstack_socket_posix.cc", "//src/core:lib/iomgr/endpoint.cc", @@ -1354,30 +1456,6 @@ grpc_cc_library( "//src/core:lib/iomgr/wakeup_fd_nospecial.cc", "//src/core:lib/iomgr/wakeup_fd_pipe.cc", "//src/core:lib/iomgr/wakeup_fd_posix.cc", - "//src/core:lib/resource_quota/api.cc", - "//src/core:lib/slice/b64.cc", - "//src/core:lib/surface/api_trace.cc", - "//src/core:lib/surface/builtins.cc", - "//src/core:lib/surface/byte_buffer.cc", - "//src/core:lib/surface/byte_buffer_reader.cc", - "//src/core:lib/surface/call.cc", - "//src/core:lib/surface/call_details.cc", - "//src/core:lib/surface/call_log_batch.cc", - "//src/core:lib/surface/call_trace.cc", - "//src/core:lib/surface/channel.cc", - "//src/core:lib/surface/channel_ping.cc", - "//src/core:lib/surface/completion_queue.cc", - "//src/core:lib/surface/completion_queue_factory.cc", - "//src/core:lib/surface/event_string.cc", - "//src/core:lib/surface/lame_client.cc", - "//src/core:lib/surface/metadata_array.cc", - "//src/core:lib/surface/server.cc", - "//src/core:lib/surface/validate_metadata.cc", - "//src/core:lib/surface/version.cc", - "//src/core:lib/surface/wait_for_cq_end_op.cc", - "//src/core:lib/transport/batch_builder.cc", - "//src/core:lib/transport/transport.cc", - "//src/core:lib/transport/transport_op_string.cc", ] + # TODO(vigneshbabu): remove these # These headers used to be vended by this target, but they have to be @@ -1388,21 +1466,8 @@ grpc_cc_library( "//src/core:lib/iomgr/event_engine_shims/tcp_client.cc", ], hdrs = [ - "//src/core:lib/channel/call_finalization.h", - "//src/core:lib/channel/call_tracer.h", - "//src/core:lib/channel/channel_stack.h", - "//src/core:lib/channel/channel_stack_builder_impl.h", - "//src/core:lib/channel/channel_trace.h", - "//src/core:lib/channel/channelz.h", - "//src/core:lib/channel/channelz_registry.h", - "//src/core:lib/channel/connected_channel.h", - "//src/core:lib/channel/promise_based_filter.h", - "//src/core:lib/channel/status_util.h", - "//src/core:lib/compression/message_compress.h", "//src/core:lib/iomgr/block_annotate.h", - "//src/core:lib/iomgr/call_combiner.h", "//src/core:lib/iomgr/cfstream_handle.h", - "//src/core:lib/iomgr/dynamic_annotations.h", "//src/core:lib/iomgr/endpoint.h", "//src/core:lib/iomgr/endpoint_cfstream.h", "//src/core:lib/iomgr/endpoint_pair.h", @@ -1442,13 +1507,166 @@ grpc_cc_library( "//src/core:lib/iomgr/vsock.h", "//src/core:lib/iomgr/wakeup_fd_pipe.h", "//src/core:lib/iomgr/wakeup_fd_posix.h", - "//src/core:lib/resource_quota/api.h", + ] + + # TODO(vigneshbabu): remove these + # These headers used to be vended by this target, but they have to be + # removed after landing EventEngine. + [ + "//src/core:lib/iomgr/event_engine_shims/closure.h", + "//src/core:lib/iomgr/event_engine_shims/endpoint.h", + "//src/core:lib/iomgr/event_engine_shims/tcp_client.h", + ], + defines = select({ + "systemd": ["HAVE_LIBSYSTEMD"], + "//conditions:default": [], + }), + external_deps = [ + "absl/base:core_headers", + "absl/cleanup", + "absl/container:flat_hash_map", + "absl/container:flat_hash_set", + "absl/container:inlined_vector", + "absl/functional:any_invocable", + "absl/functional:function_ref", + "absl/hash", + "absl/meta:type_traits", + "absl/random", + "absl/status", + "absl/status:statusor", + "absl/strings", + "absl/strings:str_format", + "absl/time", + "absl/types:optional", + "absl/utility", + "madler_zlib", + ], + language = "c++", + linkopts = select({ + "systemd": ["-lsystemd"], + "//conditions:default": [], + }), + public_hdrs = GRPC_PUBLIC_HDRS + GRPC_PUBLIC_EVENT_ENGINE_HDRS, + visibility = ["@grpc:alt_grpc_base_legacy"], + deps = [ + "channel_arg_names", + "config_vars", + "debug_location", + "dynamic_annotations", + "exec_ctx", + "gpr", + "grpc_public_hdrs", + "grpc_trace", + "iomgr_buffer_list", + "iomgr_internal_errqueue", + "iomgr_timer", + "orphanable", + "parse_address", + "ref_counted_ptr", + "resource_quota_api", + "sockaddr_utils", + "stats", + "tcp_tracer", + "//src/core:channel_args", + "//src/core:channel_args_endpoint_config", + "//src/core:closure", + "//src/core:construct_destruct", + "//src/core:context", + "//src/core:default_event_engine", + "//src/core:error", + "//src/core:error_utils", + "//src/core:event_engine_common", + "//src/core:event_engine_extensions", + "//src/core:event_engine_memory_allocator_factory", + "//src/core:event_engine_query_extensions", + "//src/core:event_engine_shim", + "//src/core:event_engine_tcp_socket_utils", + "//src/core:event_engine_trace", + "//src/core:event_log", + "//src/core:experiments", + "//src/core:gpr_atm", + "//src/core:gpr_manual_constructor", + "//src/core:gpr_spinlock", + "//src/core:grpc_sockaddr", + "//src/core:init_internally", + "//src/core:iomgr_fwd", + "//src/core:iomgr_port", + "//src/core:memory_quota", + "//src/core:no_destruct", + "//src/core:per_cpu", + "//src/core:pollset_set", + "//src/core:posix_event_engine_base_hdrs", + "//src/core:posix_event_engine_endpoint", + "//src/core:random_early_detection", + "//src/core:ref_counted", + "//src/core:resolved_address", + "//src/core:resource_quota", + "//src/core:resource_quota_trace", + "//src/core:slice", + "//src/core:slice_buffer", + "//src/core:slice_cast", + "//src/core:slice_refcount", + "//src/core:socket_mutator", + "//src/core:stats_data", + "//src/core:status_flag", + "//src/core:status_helper", + "//src/core:strerror", + "//src/core:thread_quota", + "//src/core:time", + "//src/core:useful", + "//src/core:windows_event_engine", + "//src/core:windows_event_engine_listener", + ], +) + +grpc_cc_library( + name = "grpc_base", + srcs = [ + "//src/core:lib/channel/call_tracer.cc", + "//src/core:lib/channel/channel_stack.cc", + "//src/core:lib/channel/channel_stack_builder_impl.cc", + "//src/core:lib/channel/connected_channel.cc", + "//src/core:lib/channel/promise_based_filter.cc", + "//src/core:lib/channel/server_call_tracer_filter.cc", + "//src/core:lib/channel/status_util.cc", + "//src/core:lib/compression/compression.cc", + "//src/core:lib/compression/message_compress.cc", + "//src/core:lib/slice/b64.cc", + "//src/core:lib/surface/api_trace.cc", + "//src/core:lib/surface/builtins.cc", + "//src/core:lib/surface/byte_buffer.cc", + "//src/core:lib/surface/byte_buffer_reader.cc", + "//src/core:lib/surface/call.cc", + "//src/core:lib/surface/call_details.cc", + "//src/core:lib/surface/call_log_batch.cc", + "//src/core:lib/surface/channel.cc", + "//src/core:lib/surface/channel_ping.cc", + "//src/core:lib/surface/completion_queue.cc", + "//src/core:lib/surface/completion_queue_factory.cc", + "//src/core:lib/surface/event_string.cc", + "//src/core:lib/surface/lame_client.cc", + "//src/core:lib/surface/metadata_array.cc", + "//src/core:lib/surface/server.cc", + "//src/core:lib/surface/validate_metadata.cc", + "//src/core:lib/surface/version.cc", + "//src/core:lib/surface/wait_for_cq_end_op.cc", + "//src/core:lib/transport/batch_builder.cc", + "//src/core:lib/transport/transport.cc", + "//src/core:lib/transport/transport_op_string.cc", + ], + hdrs = [ + "//src/core:lib/channel/call_finalization.h", + "//src/core:lib/channel/call_tracer.h", + "//src/core:lib/channel/channel_stack.h", + "//src/core:lib/channel/channel_stack_builder_impl.h", + "//src/core:lib/channel/connected_channel.h", + "//src/core:lib/channel/promise_based_filter.h", + "//src/core:lib/channel/status_util.h", + "//src/core:lib/compression/message_compress.h", "//src/core:lib/slice/b64.h", "//src/core:lib/surface/api_trace.h", "//src/core:lib/surface/builtins.h", "//src/core:lib/surface/call.h", "//src/core:lib/surface/call_test_only.h", - "//src/core:lib/surface/call_trace.h", "//src/core:lib/surface/channel.h", "//src/core:lib/surface/completion_queue.h", "//src/core:lib/surface/completion_queue_factory.h", @@ -1460,14 +1678,6 @@ grpc_cc_library( "//src/core:lib/surface/wait_for_cq_end_op.h", "//src/core:lib/transport/batch_builder.h", "//src/core:lib/transport/transport.h", - ] + - # TODO(vigneshbabu): remove these - # These headers used to be vended by this target, but they have to be - # removed after landing EventEngine. - [ - "//src/core:lib/iomgr/event_engine_shims/closure.h", - "//src/core:lib/iomgr/event_engine_shims/endpoint.h", - "//src/core:lib/iomgr/event_engine_shims/tcp_client.h", ], defines = select({ "systemd": ["HAVE_LIBSYSTEMD"], @@ -1501,16 +1711,21 @@ grpc_cc_library( public_hdrs = GRPC_PUBLIC_HDRS + GRPC_PUBLIC_EVENT_ENGINE_HDRS, visibility = ["@grpc:alt_grpc_base_legacy"], deps = [ + "call_combiner", + "call_trace", "channel_arg_names", "channel_stack_builder", + "channelz", "config", "config_vars", "cpp_impl_of", "debug_location", + "dynamic_annotations", "exec_ctx", "gpr", "grpc_public_hdrs", "grpc_trace", + "iomgr", "iomgr_buffer_list", "iomgr_internal_errqueue", "iomgr_timer", @@ -1519,6 +1734,7 @@ grpc_cc_library( "parse_address", "promise", "ref_counted_ptr", + "resource_quota_api", "sockaddr_utils", "stats", "tcp_tracer", @@ -1790,7 +2006,9 @@ grpc_cc_library( public_hdrs = GRPC_PUBLIC_HDRS, visibility = ["@grpc:public"], deps = [ + "call_trace", "channel_arg_names", + "channelz", "config", "debug_location", "exec_ctx", @@ -1799,9 +2017,11 @@ grpc_cc_library( "grpc_public_hdrs", "grpc_trace", "handshaker", + "iomgr", "legacy_context", "promise", "ref_counted_ptr", + "resource_quota_api", "stats", "tsi_base", "//src/core:activity", @@ -1964,9 +2184,11 @@ grpc_cc_library( "grpcpp_backend_metric_recorder", "grpcpp_call_metric_recorder", "grpcpp_status", + "iomgr", "iomgr_timer", "legacy_context", "ref_counted_ptr", + "resource_quota_api", "//src/core:arena", "//src/core:channel_args", "//src/core:channel_fwd", @@ -2040,9 +2262,11 @@ grpc_cc_library( "grpcpp_backend_metric_recorder", "grpcpp_call_metric_recorder", "grpcpp_status", + "iomgr", "iomgr_timer", "legacy_context", "ref_counted_ptr", + "resource_quota_api", "//src/core:arena", "//src/core:channel_args", "//src/core:channel_init", @@ -2602,6 +2826,7 @@ grpc_cc_library( "grpc_base", "grpc_public_hdrs", "grpc_trace", + "iomgr", "ref_counted_ptr", "//src/core:channel_args", "//src/core:closure", @@ -2638,6 +2863,7 @@ grpc_cc_library( "grpc_base", "handshaker", "httpcli", + "iomgr", "ref_counted_ptr", "//src/core:channel_args", "//src/core:closure", @@ -3074,7 +3300,9 @@ grpc_cc_library( visibility = ["@grpc:client_channel"], deps = [ "backoff", + "call_combiner", "channel_arg_names", + "channelz", "config", "config_vars", "debug_location", @@ -3088,6 +3316,7 @@ grpc_cc_library( "grpc_service_config_impl", "grpc_trace", "http_connect_handshaker", + "iomgr", "iomgr_timer", "legacy_context", "orphanable", @@ -3202,6 +3431,7 @@ grpc_cc_library( "grpc_resolver", "grpc_service_config_impl", "grpc_trace", + "iomgr", "iomgr_timer", "orphanable", "parse_address", @@ -3259,8 +3489,10 @@ grpc_cc_library( "grpc_security_base", "grpc_trace", "handshaker", + "iomgr", "orphanable", "ref_counted_ptr", + "resource_quota_api", "sockaddr_utils", "uri_parser", "//src/core:channel_args", @@ -3308,6 +3540,7 @@ grpc_cc_library( "grpc_public_hdrs", "grpc_security_base", "handshaker", + "iomgr", "promise", "ref_counted_ptr", "tsi_alts_credentials", @@ -3376,6 +3609,7 @@ grpc_cc_library( "grpc_security_base", "grpc_trace", "httpcli", + "iomgr", "orphanable", "promise", "ref_counted_ptr", @@ -3617,6 +3851,7 @@ grpc_cc_library( language = "c++", visibility = ["@grpc:http"], deps = [ + "call_trace", "channel_arg_names", "config", "gpr", @@ -4063,6 +4298,7 @@ grpc_cc_library( visibility = ["@grpc:grpclb"], deps = [ "channel_arg_names", + "channelz", "chttp2_context_list_entry", "chttp2_legacy_frame", "chttp2_varint", @@ -4077,6 +4313,7 @@ grpc_cc_library( "hpack_parser_table", "http_trace", "httpcli", + "iomgr", "iomgr_buffer_list", "legacy_context", "ref_counted_ptr", diff --git a/CMakeLists.txt b/CMakeLists.txt index 1c6288b3140..246d15f9bd4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2458,7 +2458,6 @@ add_library(grpc src/core/lib/surface/call.cc src/core/lib/surface/call_details.cc src/core/lib/surface/call_log_batch.cc - src/core/lib/surface/call_trace.cc src/core/lib/surface/channel.cc src/core/lib/surface/channel_init.cc src/core/lib/surface/channel_ping.cc @@ -3186,7 +3185,6 @@ add_library(grpc_unsecure src/core/lib/surface/call.cc src/core/lib/surface/call_details.cc src/core/lib/surface/call_log_batch.cc - src/core/lib/surface/call_trace.cc src/core/lib/surface/channel.cc src/core/lib/surface/channel_init.cc src/core/lib/surface/channel_ping.cc @@ -5311,7 +5309,6 @@ add_library(grpc_authorization_provider src/core/lib/surface/call.cc src/core/lib/surface/call_details.cc src/core/lib/surface/call_log_batch.cc - src/core/lib/surface/call_trace.cc src/core/lib/surface/channel.cc src/core/lib/surface/channel_init.cc src/core/lib/surface/channel_ping.cc diff --git a/Makefile b/Makefile index f5c020f2582..ea17f927d42 100644 --- a/Makefile +++ b/Makefile @@ -1640,7 +1640,6 @@ LIBGRPC_SRC = \ src/core/lib/surface/call.cc \ src/core/lib/surface/call_details.cc \ src/core/lib/surface/call_log_batch.cc \ - src/core/lib/surface/call_trace.cc \ src/core/lib/surface/channel.cc \ src/core/lib/surface/channel_init.cc \ src/core/lib/surface/channel_ping.cc \ @@ -2202,7 +2201,6 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/surface/call.cc \ src/core/lib/surface/call_details.cc \ src/core/lib/surface/call_log_batch.cc \ - src/core/lib/surface/call_trace.cc \ src/core/lib/surface/channel.cc \ src/core/lib/surface/channel_init.cc \ src/core/lib/surface/channel_ping.cc \ diff --git a/Package.swift b/Package.swift index 03b9b6f26b8..2eef1c50f8f 100644 --- a/Package.swift +++ b/Package.swift @@ -1769,7 +1769,6 @@ let package = Package( "src/core/lib/surface/call_details.cc", "src/core/lib/surface/call_log_batch.cc", "src/core/lib/surface/call_test_only.h", - "src/core/lib/surface/call_trace.cc", "src/core/lib/surface/call_trace.h", "src/core/lib/surface/channel.cc", "src/core/lib/surface/channel.h", diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index 3d0de3e0b37..e8e6dc1bba5 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -1914,7 +1914,6 @@ libs: - src/core/lib/surface/call.cc - src/core/lib/surface/call_details.cc - src/core/lib/surface/call_log_batch.cc - - src/core/lib/surface/call_trace.cc - src/core/lib/surface/channel.cc - src/core/lib/surface/channel_init.cc - src/core/lib/surface/channel_ping.cc @@ -2994,7 +2993,6 @@ libs: - src/core/lib/surface/call.cc - src/core/lib/surface/call_details.cc - src/core/lib/surface/call_log_batch.cc - - src/core/lib/surface/call_trace.cc - src/core/lib/surface/channel.cc - src/core/lib/surface/channel_init.cc - src/core/lib/surface/channel_ping.cc @@ -4952,7 +4950,6 @@ libs: - src/core/lib/surface/call.cc - src/core/lib/surface/call_details.cc - src/core/lib/surface/call_log_batch.cc - - src/core/lib/surface/call_trace.cc - src/core/lib/surface/channel.cc - src/core/lib/surface/channel_init.cc - src/core/lib/surface/channel_ping.cc diff --git a/config.m4 b/config.m4 index 35a684dacec..58288e0696e 100644 --- a/config.m4 +++ b/config.m4 @@ -768,7 +768,6 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/surface/call.cc \ src/core/lib/surface/call_details.cc \ src/core/lib/surface/call_log_batch.cc \ - src/core/lib/surface/call_trace.cc \ src/core/lib/surface/channel.cc \ src/core/lib/surface/channel_init.cc \ src/core/lib/surface/channel_ping.cc \ diff --git a/config.w32 b/config.w32 index c87b296783c..95c28ce6b99 100644 --- a/config.w32 +++ b/config.w32 @@ -733,7 +733,6 @@ if (PHP_GRPC != "no") { "src\\core\\lib\\surface\\call.cc " + "src\\core\\lib\\surface\\call_details.cc " + "src\\core\\lib\\surface\\call_log_batch.cc " + - "src\\core\\lib\\surface\\call_trace.cc " + "src\\core\\lib\\surface\\channel.cc " + "src\\core\\lib\\surface\\channel_init.cc " + "src\\core\\lib\\surface\\channel_ping.cc " + diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index f67f53c44b1..d0ce5cdcff6 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -1878,7 +1878,6 @@ Pod::Spec.new do |s| 'src/core/lib/surface/call_details.cc', 'src/core/lib/surface/call_log_batch.cc', 'src/core/lib/surface/call_test_only.h', - 'src/core/lib/surface/call_trace.cc', 'src/core/lib/surface/call_trace.h', 'src/core/lib/surface/channel.cc', 'src/core/lib/surface/channel.h', diff --git a/grpc.gemspec b/grpc.gemspec index a991d744105..e05517f5357 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -1771,7 +1771,6 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/surface/call_details.cc ) s.files += %w( src/core/lib/surface/call_log_batch.cc ) s.files += %w( src/core/lib/surface/call_test_only.h ) - s.files += %w( src/core/lib/surface/call_trace.cc ) s.files += %w( src/core/lib/surface/call_trace.h ) s.files += %w( src/core/lib/surface/channel.cc ) s.files += %w( src/core/lib/surface/channel.h ) diff --git a/grpc.gyp b/grpc.gyp index 20b89da75f1..defb544f104 100644 --- a/grpc.gyp +++ b/grpc.gyp @@ -954,7 +954,6 @@ 'src/core/lib/surface/call.cc', 'src/core/lib/surface/call_details.cc', 'src/core/lib/surface/call_log_batch.cc', - 'src/core/lib/surface/call_trace.cc', 'src/core/lib/surface/channel.cc', 'src/core/lib/surface/channel_init.cc', 'src/core/lib/surface/channel_ping.cc', @@ -1456,7 +1455,6 @@ 'src/core/lib/surface/call.cc', 'src/core/lib/surface/call_details.cc', 'src/core/lib/surface/call_log_batch.cc', - 'src/core/lib/surface/call_trace.cc', 'src/core/lib/surface/channel.cc', 'src/core/lib/surface/channel_init.cc', 'src/core/lib/surface/channel_ping.cc', @@ -2263,7 +2261,6 @@ 'src/core/lib/surface/call.cc', 'src/core/lib/surface/call_details.cc', 'src/core/lib/surface/call_log_batch.cc', - 'src/core/lib/surface/call_trace.cc', 'src/core/lib/surface/channel.cc', 'src/core/lib/surface/channel_init.cc', 'src/core/lib/surface/channel_ping.cc', diff --git a/package.xml b/package.xml index 7c2c650fb5d..11cea713fdd 100644 --- a/package.xml +++ b/package.xml @@ -1753,7 +1753,6 @@ - diff --git a/src/core/BUILD b/src/core/BUILD index b4174ba9988..23f5dc97499 100644 --- a/src/core/BUILD +++ b/src/core/BUILD @@ -1213,6 +1213,7 @@ grpc_cc_library( "//:gpr", "//:grpc_base", "//:handshaker", + "//:iomgr", "//:parse_address", "//:ref_counted_ptr", "//:uri_parser", @@ -3289,6 +3290,7 @@ grpc_cc_library( "//:grpc_credentials_util", "//:grpc_security_base", "//:grpc_trace", + "//:iomgr", "//:parse_address", "//:promise", "//:ref_counted_ptr", @@ -3363,8 +3365,10 @@ grpc_cc_library( "//:grpc_base", "//:grpc_security_base", "//:handshaker", + "//:iomgr", "//:promise", "//:ref_counted_ptr", + "//:resource_quota_api", "//:tsi_base", "//:tsi_fake_credentials", ], @@ -3399,6 +3403,7 @@ grpc_cc_library( "//:grpc_base", "//:grpc_security_base", "//:handshaker", + "//:iomgr", "//:promise", "//:ref_counted_ptr", "//:tsi_base", @@ -3457,6 +3462,7 @@ grpc_cc_library( "//:grpc_client_channel", "//:grpc_security_base", "//:handshaker", + "//:iomgr", "//:parse_address", "//:promise", "//:ref_counted_ptr", @@ -3499,6 +3505,7 @@ grpc_cc_library( "//:grpc_security_base", "//:grpc_trace", "//:handshaker", + "//:iomgr", "//:promise", "//:ref_counted_ptr", "//:tsi_base", @@ -3555,6 +3562,7 @@ grpc_cc_library( "//:grpc_security_base", "//:grpc_trace", "//:httpcli", + "//:iomgr", "//:orphanable", "//:ref_counted_ptr", "//:uri_parser", @@ -3627,6 +3635,7 @@ grpc_cc_library( "//:grpc_security_base", "//:grpc_trace", "//:handshaker", + "//:iomgr", "//:promise", "//:ref_counted_ptr", "//:tsi_base", @@ -3709,6 +3718,7 @@ grpc_cc_library( "//:grpc_security_base", "//:grpc_trace", "//:httpcli", + "//:iomgr", "//:orphanable", "//:promise", "//:ref_counted_ptr", @@ -3762,6 +3772,7 @@ grpc_cc_library( "//:grpc_credentials_util", "//:grpc_security_base", "//:httpcli", + "//:iomgr", "//:orphanable", "//:ref_counted_ptr", "//:uri_parser", @@ -3796,6 +3807,7 @@ grpc_cc_library( "//:grpc_base", "//:grpc_security_base", "//:handshaker", + "//:iomgr", "//:promise", "//:ref_counted_ptr", "//:tsi_base", @@ -4064,6 +4076,7 @@ grpc_cc_library( "metadata_batch", "status_helper", "time", + "//:call_combiner", "//:channel_arg_names", "//:config", "//:debug_location", @@ -4140,6 +4153,7 @@ grpc_cc_library( "slice", "slice_buffer", "validation_errors", + "//:call_trace", "//:channel_arg_names", "//:config", "//:gpr", @@ -4344,6 +4358,7 @@ grpc_cc_library( "validation_errors", "//:backoff", "//:channel_arg_names", + "//:channelz", "//:config", "//:debug_location", "//:endpoint_addresses", @@ -4441,6 +4456,7 @@ grpc_cc_library( "validation_errors", "//:backoff", "//:channel_arg_names", + "//:channelz", "//:config", "//:debug_location", "//:endpoint_addresses", @@ -4667,6 +4683,7 @@ grpc_cc_library( "//:grpc_public_hdrs", "//:grpc_security_base", "//:grpc_trace", + "//:iomgr", "//:iomgr_timer", "//:orphanable", "//:parse_address", @@ -4744,6 +4761,7 @@ grpc_cc_library( "//:grpc_security_base", "//:grpc_service_config_impl", "//:grpc_trace", + "//:iomgr", "//:parse_address", "//:ref_counted_ptr", "//:sockaddr_utils", @@ -5014,6 +5032,7 @@ grpc_cc_library( "subchannel_interface", "unique_type_name", "//:channel_arg_names", + "//:channelz", "//:debug_location", "//:exec_ctx", "//:gpr", @@ -5718,6 +5737,7 @@ grpc_cc_library( "status_helper", "//:gpr_platform", "//:grpc_base", + "//:iomgr", ], ) @@ -5758,6 +5778,7 @@ grpc_cc_library( "//:grpc_resolver", "//:grpc_service_config_impl", "//:grpc_trace", + "//:iomgr", "//:orphanable", "//:ref_counted_ptr", "//:uri_parser", @@ -5816,6 +5837,7 @@ grpc_cc_library( "//:grpc_base", "//:grpc_resolver", "//:grpc_trace", + "//:iomgr", "//:orphanable", "//:ref_counted_ptr", "//:uri_parser", @@ -6013,6 +6035,7 @@ grpc_cc_library( "//:gpr", "//:grpc_base", "//:grpc_resolver", + "//:iomgr", "//:orphanable", "//:ref_counted_ptr", "//:uri_parser", @@ -6260,6 +6283,7 @@ grpc_cc_library( "time", "unique_type_name", "//:channel_arg_names", + "//:channelz", "//:config", "//:debug_location", "//:exec_ctx", @@ -6272,6 +6296,7 @@ grpc_cc_library( "//:grpc_trace", "//:grpc_transport_chttp2", "//:handshaker", + "//:iomgr", "//:orphanable", "//:ref_counted_ptr", "//:sockaddr_utils", @@ -6312,6 +6337,7 @@ grpc_cc_library( "time", "unique_type_name", "//:channel_arg_names", + "//:channelz", "//:chttp2_legacy_frame", "//:config", "//:debug_location", @@ -6322,6 +6348,7 @@ grpc_cc_library( "//:grpc_trace", "//:grpc_transport_chttp2", "//:handshaker", + "//:iomgr", "//:orphanable", "//:ref_counted_ptr", "//:sockaddr_utils", @@ -6364,6 +6391,7 @@ grpc_cc_library( "time", "try_seq", "//:channel_arg_names", + "//:channelz", "//:config", "//:debug_location", "//:exec_ctx", @@ -6371,6 +6399,7 @@ grpc_cc_library( "//:grpc_base", "//:grpc_public_hdrs", "//:grpc_trace", + "//:iomgr", "//:promise", "//:ref_counted_ptr", ], @@ -6474,6 +6503,7 @@ grpc_cc_library( "//:grpc_security_base", "//:grpc_trace", "//:httpcli", + "//:iomgr", "//:orphanable", "//:ref_counted_ptr", "//:uri_parser", @@ -6844,6 +6874,7 @@ grpc_cc_library( ], deps = [ "1999", + "call_final_info", "for_each", "if", "latch", @@ -7000,12 +7031,14 @@ grpc_cc_library( "status_helper", "time", "try_seq", + "//:channelz", "//:gpr", "//:gpr_platform", "//:grpc_base", "//:handshaker", "//:hpack_encoder", "//:hpack_parser", + "//:iomgr", "//:orphanable", "//:ref_counted_ptr", ], @@ -7065,6 +7098,7 @@ grpc_cc_library( "//:handshaker", "//:hpack_encoder", "//:hpack_parser", + "//:iomgr", "//:ref_counted_ptr", ], ) diff --git a/src/core/lib/channel/call_tracer.h b/src/core/lib/channel/call_tracer.h index 6116a26a201..67191d1bc8a 100644 --- a/src/core/lib/channel/call_tracer.h +++ b/src/core/lib/channel/call_tracer.h @@ -30,7 +30,6 @@ #include #include "src/core/lib/channel/channel_args.h" -#include "src/core/lib/channel/channel_stack.h" #include "src/core/lib/channel/context.h" #include "src/core/lib/channel/tcp_tracer.h" #include "src/core/lib/config/core_configuration.h" diff --git a/src/core/lib/channel/channel_stack.cc b/src/core/lib/channel/channel_stack.cc index 528a299632d..ee1c1d9e943 100644 --- a/src/core/lib/channel/channel_stack.cc +++ b/src/core/lib/channel/channel_stack.cc @@ -347,3 +347,10 @@ void grpc_channel_stack::InitServerCallSpine( elem->filter->init_call(elem, call); } } + +void grpc_call_log_op(const char* file, int line, gpr_log_severity severity, + grpc_call_element* elem, + grpc_transport_stream_op_batch* op) { + gpr_log(file, line, severity, "OP[%s:%p]: %s", elem->filter->name, elem, + grpc_transport_stream_op_batch_string(op, false).c_str()); +} diff --git a/src/core/lib/channel/channel_stack_builder_impl.cc b/src/core/lib/channel/channel_stack_builder_impl.cc index 17672f73f8c..a900e4580cc 100644 --- a/src/core/lib/channel/channel_stack_builder_impl.cc +++ b/src/core/lib/channel/channel_stack_builder_impl.cc @@ -23,22 +23,162 @@ #include #include +#include +#include +#include +#include #include +#include "absl/base/thread_annotations.h" +#include "absl/container/flat_hash_map.h" #include "absl/status/status.h" +#include "absl/strings/str_cat.h" #include +#include #include "src/core/lib/channel/channel_fwd.h" #include "src/core/lib/channel/channel_stack.h" #include "src/core/lib/debug/trace.h" +#include "src/core/lib/gprpp/no_destruct.h" +#include "src/core/lib/gprpp/sync.h" +#include "src/core/lib/iomgr/closure.h" #include "src/core/lib/iomgr/error.h" +#include "src/core/lib/promise/activity.h" +#include "src/core/lib/promise/arena_promise.h" +#include "src/core/lib/promise/poll.h" #include "src/core/lib/surface/call_trace.h" #include "src/core/lib/surface/channel_stack_type.h" #include "src/core/lib/transport/error_utils.h" +#include "src/core/lib/transport/metadata_batch.h" +#include "src/core/lib/transport/transport.h" namespace grpc_core { +namespace { + +const grpc_channel_filter* PromiseTracingFilterFor( + const grpc_channel_filter* filter) { + struct DerivedFilter : public grpc_channel_filter { + explicit DerivedFilter(const grpc_channel_filter* filter) + : grpc_channel_filter{ + // start_transport_stream_op_batch: + grpc_call_next_op, + // make_call_promise: + [](grpc_channel_element* elem, CallArgs call_args, + NextPromiseFactory next_promise_factory) + -> ArenaPromise { + auto* source_filter = + static_cast(elem->filter)->filter; + gpr_log( + GPR_DEBUG, + "%s[%s] CreateCallPromise: client_initial_metadata=%s", + GetContext()->DebugTag().c_str(), + source_filter->name, + call_args.client_initial_metadata->DebugString().c_str()); + return [source_filter, child = next_promise_factory( + std::move(call_args))]() mutable { + gpr_log(GPR_DEBUG, "%s[%s] PollCallPromise: begin", + GetContext()->DebugTag().c_str(), + source_filter->name); + auto r = child(); + if (auto* p = r.value_if_ready()) { + gpr_log(GPR_DEBUG, "%s[%s] PollCallPromise: done: %s", + GetContext()->DebugTag().c_str(), + source_filter->name, (*p)->DebugString().c_str()); + } else { + gpr_log(GPR_DEBUG, "%s[%s] PollCallPromise: <>", + GetContext()->DebugTag().c_str(), + source_filter->name); + } + return r; + }; + }, + /* init_call: */ + [](grpc_channel_element* elem, CallSpineInterface* call) { + auto* source_filter = + static_cast(elem->filter)->filter; + call->client_initial_metadata().receiver.InterceptAndMap( + [source_filter](ClientMetadataHandle md) { + gpr_log(GPR_DEBUG, "%s[%s] OnClientInitialMetadata: %s", + GetContext()->DebugTag().c_str(), + source_filter->name, md->DebugString().c_str()); + return md; + }); + call->client_to_server_messages().receiver.InterceptAndMap( + [source_filter](MessageHandle msg) { + gpr_log(GPR_DEBUG, "%s[%s] OnClientToServerMessage: %s", + GetContext()->DebugTag().c_str(), + source_filter->name, msg->DebugString().c_str()); + return msg; + }); + call->server_initial_metadata().sender.InterceptAndMap( + [source_filter](ServerMetadataHandle md) { + gpr_log(GPR_DEBUG, "%s[%s] OnServerInitialMetadata: %s", + GetContext()->DebugTag().c_str(), + source_filter->name, md->DebugString().c_str()); + return md; + }); + call->server_to_client_messages().sender.InterceptAndMap( + [source_filter](MessageHandle msg) { + gpr_log(GPR_DEBUG, "%s[%s] OnServerToClientMessage: %s", + GetContext()->DebugTag().c_str(), + source_filter->name, msg->DebugString().c_str()); + return msg; + }); + call->server_trailing_metadata().sender.InterceptAndMap( + [source_filter](ServerMetadataHandle md) { + gpr_log(GPR_DEBUG, "%s[%s] OnServerTrailingMetadata: %s", + GetContext()->DebugTag().c_str(), + source_filter->name, md->DebugString().c_str()); + return md; + }); + }, + grpc_channel_next_op, + /* sizeof_call_data: */ 0, + // init_call_elem: + [](grpc_call_element*, const grpc_call_element_args*) { + return absl::OkStatus(); + }, + grpc_call_stack_ignore_set_pollset_or_pollset_set, + // destroy_call_elem: + [](grpc_call_element*, const grpc_call_final_info*, + grpc_closure*) {}, + // sizeof_channel_data: + 0, + // init_channel_elem: + [](grpc_channel_element*, grpc_channel_element_args*) { + return absl::OkStatus(); + }, + // post_init_channel_elem: + [](grpc_channel_stack*, grpc_channel_element*) {}, + // destroy_channel_elem: + [](grpc_channel_element*) {}, grpc_channel_next_get_info, + // name: + nullptr}, + filter(filter), + name_str(absl::StrCat(filter->name, ".trace")) { + this->name = name_str.c_str(); + } + const grpc_channel_filter* const filter; + const std::string name_str; + }; + struct Globals { + Mutex mu; + absl::flat_hash_map> + map ABSL_GUARDED_BY(mu); + }; + auto* globals = NoDestructSingleton::Get(); + MutexLock lock(&globals->mu); + auto it = globals->map.find(filter); + if (it != globals->map.end()) return it->second.get(); + return globals->map.emplace(filter, std::make_unique(filter)) + .first->second.get(); +} + +} // namespace + bool ChannelStackBuilderImpl::IsPromising() const { for (const auto* filter : stack()) { if (filter->make_call_promise == nullptr) return false; diff --git a/src/core/lib/surface/call_trace.cc b/src/core/lib/surface/call_trace.cc deleted file mode 100644 index 78b703313ef..00000000000 --- a/src/core/lib/surface/call_trace.cc +++ /dev/null @@ -1,163 +0,0 @@ -// Copyright 2022 gRPC authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "src/core/lib/surface/call_trace.h" - -#include -#include -#include -#include - -#include "absl/base/thread_annotations.h" -#include "absl/container/flat_hash_map.h" -#include "absl/status/status.h" -#include "absl/strings/str_cat.h" - -#include - -#include "src/core/lib/channel/channel_stack.h" -#include "src/core/lib/gprpp/no_destruct.h" -#include "src/core/lib/gprpp/sync.h" -#include "src/core/lib/iomgr/closure.h" -#include "src/core/lib/promise/activity.h" -#include "src/core/lib/promise/arena_promise.h" -#include "src/core/lib/promise/poll.h" -#include "src/core/lib/transport/metadata_batch.h" -#include "src/core/lib/transport/transport.h" - -namespace grpc_core { - -const grpc_channel_filter* PromiseTracingFilterFor( - const grpc_channel_filter* filter) { - struct DerivedFilter : public grpc_channel_filter { - explicit DerivedFilter(const grpc_channel_filter* filter) - : grpc_channel_filter{ - // start_transport_stream_op_batch: - grpc_call_next_op, - // make_call_promise: - [](grpc_channel_element* elem, CallArgs call_args, - NextPromiseFactory next_promise_factory) - -> ArenaPromise { - auto* source_filter = - static_cast(elem->filter)->filter; - gpr_log( - GPR_DEBUG, - "%s[%s] CreateCallPromise: client_initial_metadata=%s", - GetContext()->DebugTag().c_str(), - source_filter->name, - call_args.client_initial_metadata->DebugString().c_str()); - return [source_filter, child = next_promise_factory( - std::move(call_args))]() mutable { - gpr_log(GPR_DEBUG, "%s[%s] PollCallPromise: begin", - GetContext()->DebugTag().c_str(), - source_filter->name); - auto r = child(); - if (auto* p = r.value_if_ready()) { - gpr_log(GPR_DEBUG, "%s[%s] PollCallPromise: done: %s", - GetContext()->DebugTag().c_str(), - source_filter->name, (*p)->DebugString().c_str()); - } else { - gpr_log(GPR_DEBUG, "%s[%s] PollCallPromise: <>", - GetContext()->DebugTag().c_str(), - source_filter->name); - } - return r; - }; - }, - /* init_call: */ - [](grpc_channel_element* elem, CallSpineInterface* call) { - auto* source_filter = - static_cast(elem->filter)->filter; - call->client_initial_metadata().receiver.InterceptAndMap( - [source_filter](ClientMetadataHandle md) { - gpr_log(GPR_DEBUG, "%s[%s] OnClientInitialMetadata: %s", - GetContext()->DebugTag().c_str(), - source_filter->name, md->DebugString().c_str()); - return md; - }); - call->client_to_server_messages().receiver.InterceptAndMap( - [source_filter](MessageHandle msg) { - gpr_log(GPR_DEBUG, "%s[%s] OnClientToServerMessage: %s", - GetContext()->DebugTag().c_str(), - source_filter->name, msg->DebugString().c_str()); - return msg; - }); - call->server_initial_metadata().sender.InterceptAndMap( - [source_filter](ServerMetadataHandle md) { - gpr_log(GPR_DEBUG, "%s[%s] OnServerInitialMetadata: %s", - GetContext()->DebugTag().c_str(), - source_filter->name, md->DebugString().c_str()); - return md; - }); - call->server_to_client_messages().sender.InterceptAndMap( - [source_filter](MessageHandle msg) { - gpr_log(GPR_DEBUG, "%s[%s] OnServerToClientMessage: %s", - GetContext()->DebugTag().c_str(), - source_filter->name, msg->DebugString().c_str()); - return msg; - }); - call->server_trailing_metadata().sender.InterceptAndMap( - [source_filter](ServerMetadataHandle md) { - gpr_log(GPR_DEBUG, "%s[%s] OnServerTrailingMetadata: %s", - GetContext()->DebugTag().c_str(), - source_filter->name, md->DebugString().c_str()); - return md; - }); - }, - grpc_channel_next_op, - /* sizeof_call_data: */ 0, - // init_call_elem: - [](grpc_call_element*, const grpc_call_element_args*) { - return absl::OkStatus(); - }, - grpc_call_stack_ignore_set_pollset_or_pollset_set, - // destroy_call_elem: - [](grpc_call_element*, const grpc_call_final_info*, - grpc_closure*) {}, - // sizeof_channel_data: - 0, - // init_channel_elem: - [](grpc_channel_element*, grpc_channel_element_args*) { - return absl::OkStatus(); - }, - // post_init_channel_elem: - [](grpc_channel_stack*, grpc_channel_element*) {}, - // destroy_channel_elem: - [](grpc_channel_element*) {}, grpc_channel_next_get_info, - // name: - nullptr}, - filter(filter), - name_str(absl::StrCat(filter->name, ".trace")) { - this->name = name_str.c_str(); - } - const grpc_channel_filter* const filter; - const std::string name_str; - }; - struct Globals { - Mutex mu; - absl::flat_hash_map> - map ABSL_GUARDED_BY(mu); - }; - auto* globals = NoDestructSingleton::Get(); - MutexLock lock(&globals->mu); - auto it = globals->map.find(filter); - if (it != globals->map.end()) return it->second.get(); - return globals->map.emplace(filter, std::make_unique(filter)) - .first->second.get(); -} - -} // namespace grpc_core diff --git a/src/core/lib/surface/call_trace.h b/src/core/lib/surface/call_trace.h index 5a484869d02..e9abf6707e8 100644 --- a/src/core/lib/surface/call_trace.h +++ b/src/core/lib/surface/call_trace.h @@ -17,14 +17,8 @@ #include -#include "src/core/lib/channel/channel_fwd.h" #include "src/core/lib/debug/trace.h" extern grpc_core::TraceFlag grpc_call_trace; -namespace grpc_core { -const grpc_channel_filter* PromiseTracingFilterFor( - const grpc_channel_filter* filter); -} - #endif // GRPC_SRC_CORE_LIB_SURFACE_CALL_TRACE_H diff --git a/src/core/lib/transport/transport_op_string.cc b/src/core/lib/transport/transport_op_string.cc index 3f2a073a913..c11e61f7dc9 100644 --- a/src/core/lib/transport/transport_op_string.cc +++ b/src/core/lib/transport/transport_op_string.cc @@ -27,7 +27,6 @@ #include #include "src/core/lib/channel/channel_fwd.h" -#include "src/core/lib/channel/channel_stack.h" #include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/gprpp/status_helper.h" #include "src/core/lib/slice/slice_buffer.h" @@ -147,10 +146,3 @@ std::string grpc_transport_op_string(grpc_transport_op* op) { return out; } - -void grpc_call_log_op(const char* file, int line, gpr_log_severity severity, - grpc_call_element* elem, - grpc_transport_stream_op_batch* op) { - gpr_log(file, line, severity, "OP[%s:%p]: %s", elem->filter->name, elem, - grpc_transport_stream_op_batch_string(op, false).c_str()); -} diff --git a/src/cpp/ext/gcp/BUILD b/src/cpp/ext/gcp/BUILD index b7e8c08295b..e5466a4cddb 100644 --- a/src/cpp/ext/gcp/BUILD +++ b/src/cpp/ext/gcp/BUILD @@ -89,6 +89,7 @@ grpc_cc_library( "//:gpr", "//:grpc_base", "//:grpc_public_hdrs", + "//:iomgr", "//src/core:env", "//src/core:error", "//src/core:error_utils", @@ -171,6 +172,7 @@ grpc_cc_library( "//:grpc++", "//:grpc_base", "//:grpc_trace", + "//:iomgr", "//:orphanable", "//src/core:closure", "//src/core:default_event_engine", diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index ef280852032..f0827017e9f 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -742,7 +742,6 @@ CORE_SOURCE_FILES = [ 'src/core/lib/surface/call.cc', 'src/core/lib/surface/call_details.cc', 'src/core/lib/surface/call_log_batch.cc', - 'src/core/lib/surface/call_trace.cc', 'src/core/lib/surface/channel.cc', 'src/core/lib/surface/channel_init.cc', 'src/core/lib/surface/channel_ping.cc', diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index c19780fd743..f92d69c3080 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -2770,7 +2770,6 @@ src/core/lib/surface/call.h \ src/core/lib/surface/call_details.cc \ src/core/lib/surface/call_log_batch.cc \ src/core/lib/surface/call_test_only.h \ -src/core/lib/surface/call_trace.cc \ src/core/lib/surface/call_trace.h \ src/core/lib/surface/channel.cc \ src/core/lib/surface/channel.h \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 35a1d5fb44f..03621790ee7 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -2546,7 +2546,6 @@ src/core/lib/surface/call.h \ src/core/lib/surface/call_details.cc \ src/core/lib/surface/call_log_batch.cc \ src/core/lib/surface/call_test_only.h \ -src/core/lib/surface/call_trace.cc \ src/core/lib/surface/call_trace.h \ src/core/lib/surface/channel.cc \ src/core/lib/surface/channel.h \ From 53fae27f880243ac1e29d0a0d0e9eef87416aebc Mon Sep 17 00:00:00 2001 From: AJ Heller Date: Tue, 20 Feb 2024 11:59:45 -0800 Subject: [PATCH 36/37] [experiment] Disable promise_based_client_call test on Windows (#35952) There is a continuous failure in the master/windows/bazel_rbe/grpc_bazel_rbe_opt tests. These generate a lot of noise at the moment, and may be hiding other issues, so they are being disabled in CI. Closes #35952 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/35952 from drfloob:ohnowindowspbcc 39ded8e8facf1d5bd0aef79717eb381a0fd0a890 PiperOrigin-RevId: 608688522 --- bazel/experiments.bzl | 8 -------- src/core/lib/experiments/rollouts.yaml | 5 ++++- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/bazel/experiments.bzl b/bazel/experiments.bzl index 0e5e67f0028..6db7963f877 100644 --- a/bazel/experiments.bzl +++ b/bazel/experiments.bzl @@ -78,7 +78,6 @@ EXPERIMENTS = { "v3_compression_filter", ], "core_end2end_test": [ - "promise_based_client_call", "promise_based_server_call", "work_serializer_dispatch", ], @@ -97,9 +96,6 @@ EXPERIMENTS = { "tcp_frame_size_tuning", "tcp_rcv_lowat", ], - "lame_client_test": [ - "promise_based_client_call", - ], "lb_unit_test": [ "work_serializer_dispatch", ], @@ -160,7 +156,6 @@ EXPERIMENTS = { "v3_compression_filter", ], "core_end2end_test": [ - "promise_based_client_call", "promise_based_server_call", "work_serializer_dispatch", ], @@ -179,9 +174,6 @@ EXPERIMENTS = { "tcp_frame_size_tuning", "tcp_rcv_lowat", ], - "lame_client_test": [ - "promise_based_client_call", - ], "lb_unit_test": [ "work_serializer_dispatch", ], diff --git a/src/core/lib/experiments/rollouts.yaml b/src/core/lib/experiments/rollouts.yaml index 5bea6f61e1e..e0ce87097e8 100644 --- a/src/core/lib/experiments/rollouts.yaml +++ b/src/core/lib/experiments/rollouts.yaml @@ -96,7 +96,10 @@ - name: pick_first_happy_eyeballs default: true - name: promise_based_client_call - default: false + default: + ios: broken + windows: broken + posix: false - name: promise_based_server_call default: false - name: registered_method_lookup_in_transport From 014087ad4ee6813e3b3c818780e42140e65a1ba1 Mon Sep 17 00:00:00 2001 From: apolcyn Date: Tue, 20 Feb 2024 12:00:22 -0800 Subject: [PATCH 37/37] [ruby] fix interop test build (#35948) Followup to https://github.com/grpc/grpc/pull/35934 Trying to fix: ``` Required ruby-3.0.0 is not installed. To install do: 'rvm install "ruby-3.0.0"' ``` Closes #35948 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/35948 from apolcyn:fix_ri cc92b928f3a17c94b0e210361edfb58c8487b23b PiperOrigin-RevId: 608688718 --- tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh index 2558e71cddd..5dd6a42328d 100755 --- a/tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh +++ b/tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh @@ -27,7 +27,7 @@ ${name}') cp -r /var/local/jenkins/service_account $HOME || true cd /var/local/git/grpc -rvm --default use ruby-3.0 +rvm --default use ruby-3.0.5 # build Ruby interop client and server (cd src/ruby && gem install bundler -v 2.4.22 && bundle && bundle exec rake compile)