Merge branch 'master' into fix-proxy-fixture

BUILD

@@ -227,11 +227,11 @@ config_setting(
 python_config_settings()
 
 # This should be updated along with build_handwritten.yaml
-g_stands_for = "gesundheit"  # @unused
+g_stands_for = "groovy"  # @unused
 
 core_version = "44.0.0"  # @unused
 
-version = "1.67.0-dev"  # @unused
+version = "1.68.0-dev"  # @unused
 
 GPR_PUBLIC_HDRS = [
     "include/grpc/support/alloc.h",

CMakeLists.txt

@@ -25,11 +25,11 @@
 cmake_minimum_required(VERSION 3.13)
 
 set(PACKAGE_NAME          "grpc")
-set(PACKAGE_VERSION       "1.67.0-dev")
+set(PACKAGE_VERSION       "1.68.0-dev")
 set(gRPC_CORE_VERSION     "44.0.0")
 set(gRPC_CORE_SOVERSION   "44")
-set(gRPC_CPP_VERSION      "1.67.0-dev")
-set(gRPC_CPP_SOVERSION    "1.67")
+set(gRPC_CPP_VERSION      "1.68.0-dev")
+set(gRPC_CPP_SOVERSION    "1.68")
 set(PACKAGE_STRING        "${PACKAGE_NAME} ${PACKAGE_VERSION}")
 set(PACKAGE_TARNAME       "${PACKAGE_NAME}-${PACKAGE_VERSION}")
 set(PACKAGE_BUGREPORT     "https://github.com/grpc/grpc/issues/")
@@ -1591,7 +1591,7 @@ if(gRPC_BUILD_TESTS)
   if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
     add_dependencies(buildtests_cxx xds_fallback_end2end_test)
   endif()
-  if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
+  if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_POSIX)
     add_dependencies(buildtests_cxx xds_fault_injection_end2end_test)
   endif()
   if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
@@ -35482,7 +35482,7 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
 endif()
 endif()
 if(gRPC_BUILD_TESTS)
-if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
+if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_POSIX)
 
   add_executable(xds_fault_injection_end2end_test
     ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/duplicate/echo_duplicate.pb.cc

Makefile

@@ -368,7 +368,7 @@ Q = @
 endif
 
 CORE_VERSION = 44.0.0
-CPP_VERSION = 1.67.0-dev
+CPP_VERSION = 1.68.0-dev
 
 CPPFLAGS_NO_ARCH += $(addprefix -I, $(INCLUDES)) $(addprefix -D, $(DEFINES))
 CPPFLAGS += $(CPPFLAGS_NO_ARCH) $(ARCH_FLAGS)

_metadata.py

@@ -14,4 +14,4 @@
 
 # AUTO-GENERATED FROM `$REPO_ROOT/templates/_metadata.py.template`!!!
 
-__version__ = """1.67.0.dev0"""
+__version__ = """1.68.0.dev0"""

build_autogenerated.yaml

@@ -21871,7 +21871,6 @@ targets:
   platforms:
   - linux
   - posix
-  - mac
 - name: xds_gcp_authn_end2end_test
   gtest: true
   build: test

build_handwritten.yaml

@@ -14,9 +14,9 @@ settings:
   '#10': See the expand_version.py for all the quirks here
   core_version: 44.0.0
   csharp_major_version: 2
-  g_stands_for: gesundheit
+  g_stands_for: groovy
   protobuf_version: 3.27.2
-  version: 1.67.0-dev
+  version: 1.68.0-dev
 configs:
   asan:
     CC: clang

config.m4

@@ -1400,7 +1400,7 @@ if test "$PHP_GRPC" != "no"; then
     -D_HAS_EXCEPTIONS=0 -DNOMINMAX -DGRPC_ARES=0 \
     -DGRPC_POSIX_FORK_ALLOW_PTHREAD_ATFORK=1 \
     -DGRPC_XDS_USER_AGENT_NAME_SUFFIX='"\"PHP\""' \
-    -DGRPC_XDS_USER_AGENT_VERSION_SUFFIX='"\"1.67.0dev\""')
+    -DGRPC_XDS_USER_AGENT_VERSION_SUFFIX='"\"1.68.0dev\""')
 
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/channelz)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/client_channel)

doc/g_stands_for.md

@@ -66,4 +66,5 @@
 - 1.64 'g' stands for ['grateful'](https://github.com/grpc/grpc/tree/v1.64.x)
 - 1.65 'g' stands for ['gnarly'](https://github.com/grpc/grpc/tree/v1.65.x)
 - 1.66 'g' stands for ['gladiator'](https://github.com/grpc/grpc/tree/v1.66.x)
-- 1.67 'g' stands for ['gesundheit'](https://github.com/grpc/grpc/tree/master)
+- 1.67 'g' stands for ['gesundheit'](https://github.com/grpc/grpc/tree/v1.67.x)
+- 1.68 'g' stands for ['groovy'](https://github.com/grpc/grpc/tree/master)

gRPC-C++.podspec

@@ -22,7 +22,7 @@
 Pod::Spec.new do |s|
   s.name     = 'gRPC-C++'
   # TODO (mxyan): use version that match gRPC version when pod is stabilized
-  version = '1.67.0-dev'
+  version = '1.68.0-dev'
   s.version  = version
   s.summary  = 'gRPC C++ library'
   s.homepage = 'https://grpc.io'

gRPC-Core.podspec

@@ -21,7 +21,7 @@
 
 Pod::Spec.new do |s|
   s.name     = 'gRPC-Core'
-  version = '1.67.0-dev'
+  version = '1.68.0-dev'
   s.version  = version
   s.summary  = 'Core cross-platform gRPC library, written in C'
   s.homepage = 'https://grpc.io'

gRPC-ProtoRPC.podspec

@@ -21,7 +21,7 @@
 
 Pod::Spec.new do |s|
   s.name     = 'gRPC-ProtoRPC'
-  version = '1.67.0-dev'
+  version = '1.68.0-dev'
   s.version  = version
   s.summary  = 'RPC library for Protocol Buffers, based on gRPC'
   s.homepage = 'https://grpc.io'

gRPC-RxLibrary.podspec

@@ -21,7 +21,7 @@
 
 Pod::Spec.new do |s|
   s.name     = 'gRPC-RxLibrary'
-  version = '1.67.0-dev'
+  version = '1.68.0-dev'
   s.version  = version
   s.summary  = 'Reactive Extensions library for iOS/OSX.'
   s.homepage = 'https://grpc.io'

gRPC.podspec

@@ -20,7 +20,7 @@
 
 Pod::Spec.new do |s|
   s.name     = 'gRPC'
-  version = '1.67.0-dev'
+  version = '1.68.0-dev'
   s.version  = version
   s.summary  = 'gRPC client library for iOS/OSX'
   s.homepage = 'https://grpc.io'

include/grpcpp/version_info.h

@@ -19,9 +19,9 @@
 #define GRPCPP_VERSION_INFO_H
 
 #define GRPC_CPP_VERSION_MAJOR 1
-#define GRPC_CPP_VERSION_MINOR 67
+#define GRPC_CPP_VERSION_MINOR 68
 #define GRPC_CPP_VERSION_PATCH 0
 #define GRPC_CPP_VERSION_TAG "dev"
-#define GRPC_CPP_VERSION_STRING "1.67.0-dev"
+#define GRPC_CPP_VERSION_STRING "1.68.0-dev"
 
 #endif  // GRPCPP_VERSION_INFO_H

package.xml

@@ -13,8 +13,8 @@
  <date>2019-09-24</date>
  <time>16:06:07</time>
  <version>
-  <release>1.67.0dev</release>
-  <api>1.67.0dev</api>
+  <release>1.68.0dev</release>
+  <api>1.68.0dev</api>
  </version>
  <stability>
   <release>beta</release>
@@ -22,7 +22,7 @@
  </stability>
  <license>Apache 2.0</license>
  <notes>
-- gRPC Core 1.67.0 update
+- gRPC Core 1.68.0 update
  </notes>
  <contents>
   <dir baseinstalldir="/" name="/">

src/core/client_channel/client_channel_filter.cc

@@ -2518,16 +2518,17 @@ ClientChannelFilter::LoadBalancedCall::PickSubchannel(bool was_queued) {
   // updated before we queue it.
   // We need to unref pickers in the WorkSerializer.
   std::vector<RefCountedPtr<LoadBalancingPolicy::SubchannelPicker>> pickers;
-  auto cleanup = absl::MakeCleanup([&]() {
-    if (IsWorkSerializerDispatchEnabled()) return;
-    chand_->work_serializer_->Run(
-        [pickers = std::move(pickers)]() mutable {
-          for (auto& picker : pickers) {
-            picker.reset(DEBUG_LOCATION, "PickSubchannel");
-          }
-        },
-        DEBUG_LOCATION);
-  });
+  auto cleanup = absl::MakeCleanup(
+      [work_serializer = chand_->work_serializer_, &pickers]() {
+        if (IsWorkSerializerDispatchEnabled()) return;
+        work_serializer->Run(
+            [pickers = std::move(pickers)]() mutable {
+              for (auto& picker : pickers) {
+                picker.reset(DEBUG_LOCATION, "PickSubchannel");
+              }
+            },
+            DEBUG_LOCATION);
+      });
   absl::AnyInvocable<void(RefCountedPtr<LoadBalancingPolicy::SubchannelPicker>)>
       set_picker;
   if (!IsWorkSerializerDispatchEnabled()) {

src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc

@@ -187,11 +187,7 @@ void ChaoticGoodServerListener::ActiveConnection::NewConnectionID() {
       connection_id_, std::make_shared<InterActivityLatch<PromiseEndpoint>>());
 }
 
-void ChaoticGoodServerListener::ActiveConnection::Done(
-    absl::optional<absl::string_view> error) {
-  if (error.has_value()) {
-    LOG(ERROR) << "ActiveConnection::Done:" << this << " " << *error;
-  }
+void ChaoticGoodServerListener::ActiveConnection::Done() {
   // Can easily be holding various locks here: bounce through EE to ensure no
   // deadlocks.
   listener_->event_engine_->Run([self = Ref()]() {
@@ -387,13 +383,15 @@ auto ChaoticGoodServerListener::ActiveConnection::HandshakingState::
 void ChaoticGoodServerListener::ActiveConnection::HandshakingState::
     OnHandshakeDone(absl::StatusOr<HandshakerArgs*> result) {
   if (!result.ok()) {
-    connection_->Done(
-        absl::StrCat("Handshake failed: ", result.status().ToString()));
+    LOG_EVERY_N_SEC(ERROR, 5) << "Handshake failed: ", result.status();
+    connection_->Done();
     return;
   }
   CHECK_NE(*result, nullptr);
   if ((*result)->endpoint == nullptr) {
-    connection_->Done("Server handshake done but has empty endpoint.");
+    LOG_EVERY_N_SEC(ERROR, 5)
+        << "Server handshake done but has empty endpoint.";
+    connection_->Done();
     return;
   }
   CHECK(grpc_event_engine::experimental::grpc_is_event_engine_endpoint(
@@ -429,12 +427,10 @@ void ChaoticGoodServerListener::ActiveConnection::HandshakingState::
       EventEngineWakeupScheduler(connection_->listener_->event_engine_),
       [self = Ref()](absl::Status status) {
         if (!status.ok()) {
-          self->connection_->Done(
-              absl::StrCat("Server setting frame handling failed: ",
-                           StatusToString(status)));
-        } else {
-          self->connection_->Done();
+          GRPC_TRACE_LOG(chaotic_good, ERROR)
+              << "Server setting frame handling failed: " << status;
         }
+        self->connection_->Done();
       },
       connection_->arena_.get());
   MutexLock lock(&connection_->mu_);

src/core/ext/transport/chaotic_good/server/chaotic_good_server.h

@@ -111,7 +111,7 @@ class ChaoticGoodServerListener final : public Server::ListenerInterface {
     };
 
    private:
-    void Done(absl::optional<absl::string_view> error = absl::nullopt);
+    void Done();
     void NewConnectionID();
     RefCountedPtr<Arena> arena_ = SimpleArenaAllocator()->MakeArena();
     const RefCountedPtr<ChaoticGoodServerListener> listener_;

src/core/ext/transport/chaotic_good/server_transport.cc

@@ -237,10 +237,11 @@ auto ChaoticGoodServerTransport::DeserializeAndPushFragmentToNewCall(
     call_initiator.emplace(std::move(call.initiator));
     auto add_result = NewStream(frame_header.stream_id, *call_initiator);
     if (add_result.ok()) {
-      call_destination_->StartCall(std::move(call.handler));
       call_initiator->SpawnGuarded(
           "server-write", [this, stream_id = frame_header.stream_id,
-                           call_initiator = *call_initiator]() {
+                           call_initiator = *call_initiator,
+                           call_handler = std::move(call.handler)]() mutable {
+            call_destination_->StartCall(std::move(call_handler));
             return CallOutboundLoop(stream_id, call_initiator);
           });
     } else {

src/core/lib/event_engine/posix_engine/posix_endpoint.cc

@@ -236,7 +236,7 @@ msg_iovlen_type TcpZerocopySendRecord::PopulateIovs(size_t* unwind_slice_idx,
        iov_size++) {
     MutableSlice& slice = internal::SliceCast<MutableSlice>(
         buf_.MutableSliceAt(out_offset_.slice_idx));
-    iov[iov_size].iov_base = slice.begin();
+    iov[iov_size].iov_base = slice.begin() + out_offset_.byte_idx;
     iov[iov_size].iov_len = slice.length() - out_offset_.byte_idx;
     *sending_length += iov[iov_size].iov_len;
     ++(out_offset_.slice_idx);

src/core/lib/gprpp/time.h

@@ -296,6 +296,14 @@ class Duration {
   int64_t millis_;
 };
 
+inline std::ostream& operator<<(std::ostream& out, const Duration& d) {
+  return out << d.ToString();
+}
+
+inline std::ostream& operator<<(std::ostream& out, const Timestamp& d) {
+  return out << d.ToString();
+}
+
 inline Duration operator+(Duration lhs, Duration rhs) {
   return Duration::Milliseconds(
       time_detail::MillisAdd(lhs.millis(), rhs.millis()));

src/core/lib/promise/party.cc

@@ -41,18 +41,18 @@ namespace grpc_core {
 // PartySyncUsingAtomics
 
 GRPC_MUST_USE_RESULT bool Party::RefIfNonZero() {
-  auto count = state_.load(std::memory_order_relaxed);
+  auto state = state_.load(std::memory_order_relaxed);
   do {
     // If zero, we are done (without an increment). If not, we must do a CAS
     // to maintain the contract: do not increment the counter if it is already
     // zero
-    if (count == 0) {
+    if ((state & kRefMask) == 0) {
       return false;
     }
-  } while (!state_.compare_exchange_weak(count, count + kOneRef,
+  } while (!state_.compare_exchange_weak(state, state + kOneRef,
                                          std::memory_order_acq_rel,
                                          std::memory_order_relaxed));
-  LogStateChange("RefIfNonZero", count, count + kOneRef);
+  LogStateChange("RefIfNonZero", state, state + kOneRef);
   return true;
 }
 

src/core/lib/resource_quota/periodic_update.cc

@@ -68,8 +68,8 @@ bool PeriodicUpdate::MaybeEndPeriod(absl::FunctionRef<void(Duration)> f) {
   expected_updates_per_period_ =
       period_.seconds() * expected_updates_per_period_ / time_so_far.seconds();
   if (expected_updates_per_period_ < 1) expected_updates_per_period_ = 1;
-  period_start_ = now;
   f(time_so_far);
+  period_start_ = Timestamp::Now();
   updates_remaining_.store(expected_updates_per_period_,
                            std::memory_order_release);
   return true;

src/core/lib/surface/version.cc

@@ -24,4 +24,4 @@
 
 const char* grpc_version_string(void) { return "44.0.0"; }
 
-const char* grpc_g_stands_for(void) { return "gesundheit"; }
+const char* grpc_g_stands_for(void) { return "groovy"; }

src/core/server/xds_server_config_fetcher.cc

@@ -677,8 +677,16 @@ void XdsServerConfigFetcher::ListenerWatcher::
     // It should get cleaned up eventually. Ignore this update.
     return;
   }
+  bool first_good_update = filter_chain_match_manager_ == nullptr;
+  // Promote the pending FilterChainMatchManager
+  filter_chain_match_manager_ = std::move(pending_filter_chain_match_manager_);
+  // TODO(yashykt): Right now, the server_config_watcher_ does not invoke
+  // XdsServerConfigFetcher while holding a lock, but that might change in the
+  // future in which case we would want to execute this update outside the
+  // critical region through a WorkSerializer similar to XdsClient.
+  server_config_watcher_->UpdateConnectionManager(filter_chain_match_manager_);
   // Let the logger know about the update if there was no previous good update.
-  if (filter_chain_match_manager_ == nullptr) {
+  if (first_good_update) {
     if (serving_status_notifier_.on_serving_status_update != nullptr) {
       serving_status_notifier_.on_serving_status_update(
           serving_status_notifier_.user_data, listening_address_.c_str(),
@@ -688,13 +696,6 @@ void XdsServerConfigFetcher::ListenerWatcher::
                 << listening_address_;
     }
   }
-  // Promote the pending FilterChainMatchManager
-  filter_chain_match_manager_ = std::move(pending_filter_chain_match_manager_);
-  // TODO(yashykt): Right now, the server_config_watcher_ does not invoke
-  // XdsServerConfigFetcher while holding a lock, but that might change in the
-  // future in which case we would want to execute this update outside the
-  // critical region through a WorkSerializer similar to XdsClient.
-  server_config_watcher_->UpdateConnectionManager(filter_chain_match_manager_);
 }
 
 //

src/core/xds/xds_client/xds_client.cc

@@ -210,6 +210,7 @@ class XdsClient::XdsChannel::AdsCall final
       if (timer_handle_.has_value() &&
           ads_call_->xds_client()->engine()->Cancel(*timer_handle_)) {
         timer_handle_.reset();
+        ads_call_.reset();
       }
     }
 
@@ -250,24 +251,28 @@ class XdsClient::XdsChannel::AdsCall final
     }
 
     void OnTimer() {
-      GRPC_TRACE_LOG(xds_client, INFO)
-          << "[xds_client " << ads_call_->xds_client() << "] xds server "
-          << ads_call_->xds_channel()->server_.server_uri()
-          << ": timeout obtaining resource {type=" << type_->type_url()
-          << " name="
-          << XdsClient::ConstructFullXdsResourceName(
-                 name_.authority, type_->type_url(), name_.key)
-          << "} from xds server";
       {
         MutexLock lock(&ads_call_->xds_client()->mu_);
         timer_handle_.reset();
-        resource_seen_ = true;
         auto& authority_state =
             ads_call_->xds_client()->authority_state_map_[name_.authority];
         ResourceState& state = authority_state.resource_map[type_][name_.key];
-        state.meta.client_status = XdsApi::ResourceMetadata::DOES_NOT_EXIST;
-        ads_call_->xds_client()->NotifyWatchersOnResourceDoesNotExist(
-            state.watchers, ReadDelayHandle::NoWait());
+        // We might have received the resource after the timer fired but before
+        // the callback ran.
+        if (state.resource == nullptr) {
+          GRPC_TRACE_LOG(xds_client, INFO)
+              << "[xds_client " << ads_call_->xds_client() << "] xds server "
+              << ads_call_->xds_channel()->server_.server_uri()
+              << ": timeout obtaining resource {type=" << type_->type_url()
+              << " name="
+              << XdsClient::ConstructFullXdsResourceName(
+                     name_.authority, type_->type_url(), name_.key)
+              << "} from xds server";
+          resource_seen_ = true;
+          state.meta.client_status = XdsApi::ResourceMetadata::DOES_NOT_EXIST;
+          ads_call_->xds_client()->NotifyWatchersOnResourceDoesNotExist(
+              state.watchers, ReadDelayHandle::NoWait());
+        }
       }
       ads_call_->xds_client()->work_serializer_.DrainQueue();
       ads_call_.reset();

src/csharp/build/dependencies.props

@@ -1,7 +1,7 @@
 <!-- This file is generated -->
 <Project>
   <PropertyGroup>
-    <GrpcCsharpVersion>2.67.0-dev</GrpcCsharpVersion>
+    <GrpcCsharpVersion>2.68.0-dev</GrpcCsharpVersion>
     <GoogleProtobufVersion>3.27.2</GoogleProtobufVersion>
   </PropertyGroup>
 </Project>

src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec

@@ -42,7 +42,7 @@ Pod::Spec.new do |s|
   # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed
   # before them.
   s.name     = '!ProtoCompiler-gRPCCppPlugin'
-  v = '1.67.0-dev'
+  v = '1.68.0-dev'
   s.version  = v
   s.summary  = 'The gRPC ProtoC plugin generates C++ files from .proto services.'
   s.description = <<-DESC

src/objective-c/!ProtoCompiler-gRPCPlugin.podspec

@@ -42,7 +42,7 @@ Pod::Spec.new do |s|
   # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed
   # before them.
   s.name     = '!ProtoCompiler-gRPCPlugin'
-  v = '1.67.0-dev'
+  v = '1.68.0-dev'
   s.version  = v
   s.summary  = 'The gRPC ProtoC plugin generates Objective-C files from .proto services.'
   s.description = <<-DESC

src/objective-c/BoringSSL-GRPC.podspec

@@ -132,7 +132,7 @@ Pod::Spec.new do |s|
     ss.source_files = 'src/ssl/*.{h,c,cc}',
                       'src/ssl/**/*.{h,c,cc}',
                       'src/crypto/*.{h,c,cc}',
-                      'src/crypto/**/*.{h,c,cc}',
+                      'src/crypto/**/*.{h,c,cc,inc}',
                       # We have to include fiat because spake25519 depends on it
                       'src/third_party/fiat/*.{h,c,cc}',
                       # Include the err_data.c pre-generated in boringssl's master-with-bazel branch
@@ -143,11 +143,7 @@ Pod::Spec.new do |s|
                               'src/crypto/*.h',
                               'src/crypto/**/*.h',
                               'src/third_party/fiat/*.h'
-    # bcm.c includes other source files, creating duplicated symbols. Since it is not used, we
-    # explicitly exclude it from the pod.
-    # TODO (mxyan): Work with BoringSSL team to remove this hack.
-    ss.exclude_files = 'src/crypto/fipsmodule/bcm.c',
-                       'src/**/*_test.*',
+    ss.exclude_files = 'src/**/*_test.*',
                        'src/**/test_*.*',
                        'src/**/test/*.*'
 
@@ -713,10 +709,10 @@ Pod::Spec.new do |s|
     EOF
 
     # We are renaming openssl to openssl_grpc so that there is no conflict with openssl if it exists
-    find . -type f \\( -path '*.h' -or -path '*.cc' -or -path '*.c' \\) -print0 | xargs -0 -L1 sed -E -i'.grpc_back' 's;#include <openssl/;#include <openssl_grpc/;g'
+    find . -type f \\( -path '*.h' -or -path '*.cc' -or -path '*.c' -or -path '*.inc' \\) -print0 | xargs -0 -L1 sed -E -i'.grpc_back' 's;#include <openssl/;#include <openssl_grpc/;g'
 
     # Include of boringssl_prefix_symbols.h does not follow Xcode import style. We add the package
     # name here so that Xcode knows where to find it.
-    find . -type f \\( -path '*.h' -or -path '*.cc' -or -path '*.c' \\) -print0 | xargs -0 -L1 sed -E -i'.grpc_back' 's;#include <boringssl_prefix_symbols.h>;#include <openssl_grpc/boringssl_prefix_symbols.h>;g'
+    find . -type f \\( -path '*.h' -or -path '*.cc' -or -path '*.c' -or -path '*.inc' \\) -print0 | xargs -0 -L1 sed -E -i'.grpc_back' 's;#include <boringssl_prefix_symbols.h>;#include <openssl_grpc/boringssl_prefix_symbols.h>;g'
   END_OF_COMMAND
 end

src/objective-c/GRPCClient/version.h

@@ -22,4 +22,4 @@
 // instead. This file can be regenerated from the template by running
 // `tools/buildgen/generate_projects.sh`.
 
-#define GRPC_OBJC_VERSION_STRING @"1.67.0-dev"
+#define GRPC_OBJC_VERSION_STRING @"1.68.0-dev"

src/objective-c/tests/Podfile

@@ -22,7 +22,7 @@ def grpc_deps
 end
 
 target 'TvTests' do
-  platform :tvos, '10.0'
+  platform :tvos, '12.0'
   grpc_deps
 end
 

src/objective-c/tests/version.h

@@ -22,5 +22,5 @@
 // instead. This file can be regenerated from the template by running
 // `tools/buildgen/generate_projects.sh`.
 
-#define GRPC_OBJC_VERSION_STRING @"1.67.0-dev"
+#define GRPC_OBJC_VERSION_STRING @"1.68.0-dev"
 #define GRPC_C_VERSION_STRING @"44.0.0"

src/php/composer.json

@@ -2,7 +2,7 @@
   "name": "grpc/grpc-dev",
   "description": "gRPC library for PHP - for Development use only",
   "license": "Apache-2.0",
-  "version": "1.67.0",
+  "version": "1.68.0",
   "require": {
     "php": ">=7.0.0",
     "google/protobuf": "^v3.3.0"

src/php/ext/grpc/version.h

@@ -20,6 +20,6 @@
 #ifndef VERSION_H
 #define VERSION_H
 
-#define PHP_GRPC_VERSION "1.67.0dev"
+#define PHP_GRPC_VERSION "1.68.0dev"
 
 #endif /* VERSION_H */

src/python/grpcio/grpc/_grpcio_metadata.py

@@ -14,4 +14,4 @@
 
 # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc/_grpcio_metadata.py.template`!!!
 
-__version__ = """1.67.0.dev0"""
+__version__ = """1.68.0.dev0"""

src/python/grpcio/grpc_version.py

@@ -14,4 +14,4 @@
 
 # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_version.py.template`!!!
 
-VERSION = '1.67.0.dev0'
+VERSION = '1.68.0.dev0'

src/python/grpcio_admin/grpc_version.py

@@ -14,4 +14,4 @@
 
 # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_admin/grpc_version.py.template`!!!
 
-VERSION = '1.67.0.dev0'
+VERSION = '1.68.0.dev0'

src/python/grpcio_channelz/grpc_version.py

@@ -14,4 +14,4 @@
 
 # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_channelz/grpc_version.py.template`!!!
 
-VERSION = '1.67.0.dev0'
+VERSION = '1.68.0.dev0'

src/python/grpcio_csds/grpc_version.py

@@ -14,4 +14,4 @@
 
 # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_csds/grpc_version.py.template`!!!
 
-VERSION = '1.67.0.dev0'
+VERSION = '1.68.0.dev0'

src/python/grpcio_csm_observability/grpc_version.py

@@ -14,4 +14,4 @@
 
 # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_csm_observability/grpc_version.py.template`!!!
 
-VERSION = '1.67.0.dev0'
+VERSION = '1.68.0.dev0'

src/python/grpcio_health_checking/grpc_version.py

@@ -14,4 +14,4 @@
 
 # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_health_checking/grpc_version.py.template`!!!
 
-VERSION = '1.67.0.dev0'
+VERSION = '1.68.0.dev0'

src/python/grpcio_observability/grpc_version.py

@@ -14,4 +14,4 @@
 
 # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_observability/grpc_version.py.template`!!!
 
-VERSION = '1.67.0.dev0'
+VERSION = '1.68.0.dev0'

src/python/grpcio_reflection/grpc_version.py

@@ -14,4 +14,4 @@
 
 # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_reflection/grpc_version.py.template`!!!
 
-VERSION = '1.67.0.dev0'
+VERSION = '1.68.0.dev0'

src/python/grpcio_status/grpc_version.py

@@ -14,4 +14,4 @@
 
 # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_status/grpc_version.py.template`!!!
 
-VERSION = '1.67.0.dev0'
+VERSION = '1.68.0.dev0'

src/python/grpcio_testing/grpc_version.py

@@ -14,4 +14,4 @@
 
 # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_testing/grpc_version.py.template`!!!
 
-VERSION = '1.67.0.dev0'
+VERSION = '1.68.0.dev0'

src/python/grpcio_tests/grpc_version.py

@@ -14,4 +14,4 @@
 
 # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_tests/grpc_version.py.template`!!!
 
-VERSION = '1.67.0.dev0'
+VERSION = '1.68.0.dev0'

src/ruby/lib/grpc/version.rb

@@ -14,5 +14,5 @@
 
 # GRPC contains the General RPC module.
 module GRPC
-  VERSION = '1.67.0.dev'
+  VERSION = '1.68.0.dev'
 end

src/ruby/nativedebug/version.rb

@@ -14,6 +14,6 @@
 
 module GRPC
   module NativeDebug
-    VERSION = '1.67.0.dev'
+    VERSION = '1.68.0.dev'
   end
 end

src/ruby/tools/version.rb

@@ -14,6 +14,6 @@
 
 module GRPC
   module Tools
-    VERSION = '1.67.0.dev'
+    VERSION = '1.68.0.dev'
   end
 end

templates/src/objective-c/BoringSSL-GRPC.podspec.template

@@ -163,7 +163,7 @@
       ss.source_files = 'src/ssl/*.{h,c,cc}',
                         'src/ssl/**/*.{h,c,cc}',
                         'src/crypto/*.{h,c,cc}',
-                        'src/crypto/**/*.{h,c,cc}',
+                        'src/crypto/**/*.{h,c,cc,inc}',
                         # We have to include fiat because spake25519 depends on it
                         'src/third_party/fiat/*.{h,c,cc}',
                         # Include the err_data.c pre-generated in boringssl's master-with-bazel branch
@@ -174,11 +174,7 @@
                                 'src/crypto/*.h',
                                 'src/crypto/**/*.h',
                                 'src/third_party/fiat/*.h'
-      # bcm.c includes other source files, creating duplicated symbols. Since it is not used, we
-      # explicitly exclude it from the pod.
-      # TODO (mxyan): Work with BoringSSL team to remove this hack.
-      ss.exclude_files = 'src/crypto/fipsmodule/bcm.c',
-                         'src/**/*_test.*',
+      ss.exclude_files = 'src/**/*_test.*',
                          'src/**/test_*.*',
                          'src/**/test/*.*'
 
@@ -221,10 +217,10 @@
       EOF
 
       # We are renaming openssl to openssl_grpc so that there is no conflict with openssl if it exists
-      find . -type f \\( -path '*.h' -or -path '*.cc' -or -path '*.c' \\) -print0 | xargs -0 -L1 sed -E -i'.grpc_back' 's;#include <openssl/;#include <openssl_grpc/;g'
+      find . -type f \\( -path '*.h' -or -path '*.cc' -or -path '*.c' -or -path '*.inc' \\) -print0 | xargs -0 -L1 sed -E -i'.grpc_back' 's;#include <openssl/;#include <openssl_grpc/;g'
 
       # Include of boringssl_prefix_symbols.h does not follow Xcode import style. We add the package
       # name here so that Xcode knows where to find it.
-      find . -type f \\( -path '*.h' -or -path '*.cc' -or -path '*.c' \\) -print0 | xargs -0 -L1 sed -E -i'.grpc_back' 's;#include <boringssl_prefix_symbols.h>;#include <openssl_grpc/boringssl_prefix_symbols.h>;g'
+      find . -type f \\( -path '*.h' -or -path '*.cc' -or -path '*.c' -or -path '*.inc' \\) -print0 | xargs -0 -L1 sed -E -i'.grpc_back' 's;#include <boringssl_prefix_symbols.h>;#include <openssl_grpc/boringssl_prefix_symbols.h>;g'
     END_OF_COMMAND
   end

test/core/end2end/connection_refused_test.cc

@@ -86,7 +86,8 @@ static void run_test(bool wait_for_ready, bool use_service_config) {
   chan = grpc_channel_create(addr.c_str(), creds, args);
   grpc_channel_credentials_release(creds);
   grpc_slice host = grpc_slice_from_static_string("nonexistant");
-  gpr_timespec deadline = grpc_timeout_seconds_to_deadline(2);
+  gpr_timespec deadline =
+      grpc_timeout_seconds_to_deadline(wait_for_ready ? 2 : 600);
   call =
       grpc_channel_create_call(chan, nullptr, GRPC_PROPAGATE_DEFAULTS, cq,
                                grpc_slice_from_static_string("/service/method"),

test/core/test_util/BUILD

@@ -395,7 +395,10 @@ grpc_cc_library(
     testonly = True,
     srcs = ["socket_use_after_close_detector.cc"],
     hdrs = ["socket_use_after_close_detector.h"],
-    external_deps = ["gtest"],
+    external_deps = [
+        "absl/log:log",
+        "gtest",
+    ],
     language = "C++",
     deps = [
         "grpc_test_util",

test/core/test_util/socket_use_after_close_detector.cc

@@ -33,6 +33,7 @@
 #include <thread>
 #include <vector>
 
+#include "absl/log/log.h"
 #include "gtest/gtest.h"
 
 #include <grpc/support/sync.h>
@@ -40,71 +41,17 @@
 #include "src/core/lib/iomgr/sockaddr.h"
 #include "test/core/test_util/port.h"
 
-// TODO(unknown): pull in different headers when enabling this
-// test on windows. Also set BAD_SOCKET_RETURN_VAL
-// to INVALID_SOCKET on windows.
-#ifdef GPR_WINDOWS
-#include "src/core/lib/iomgr/socket_windows.h"
-#include "src/core/lib/iomgr/tcp_windows.h"
-
-#define BAD_SOCKET_RETURN_VAL INVALID_SOCKET
-#else
 #define BAD_SOCKET_RETURN_VAL (-1)
-#endif
 
 namespace {
 
 #ifdef GPR_WINDOWS
 void OpenAndCloseSocketsStressLoop(int port, gpr_event* done_ev) {
-  sockaddr_in6 addr;
-  memset(&addr, 0, sizeof(addr));
-  addr.sin6_family = AF_INET6;
-  addr.sin6_port = htons(port);
-  ((char*)&addr.sin6_addr)[15] = 1;
-  for (;;) {
-    if (gpr_event_get(done_ev)) {
-      return;
-    }
-    std::vector<int> sockets;
-    for (size_t i = 0; i < 50; i++) {
-      SOCKET s = WSASocket(AF_INET6, SOCK_STREAM, IPPROTO_TCP, nullptr, 0,
-                           WSA_FLAG_OVERLAPPED);
-      ASSERT_TRUE(s != BAD_SOCKET_RETURN_VAL)
-          << "Failed to create TCP ipv6 socket";
-      char val = 1;
-      ASSERT_TRUE(setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val)) !=
-                  SOCKET_ERROR)
-          << "Failed to set socketopt reuseaddr. WSA error: " +
-                 std::to_string(WSAGetLastError());
-      ASSERT_TRUE(grpc_tcp_set_non_block(s) == absl::OkStatus())
-          << "Failed to set socket non-blocking";
-      ASSERT_TRUE(bind(s, (const sockaddr*)&addr, sizeof(addr)) != SOCKET_ERROR)
-          << "Failed to bind socket " + std::to_string(s) +
-                 " to [::1]:" + std::to_string(port) +
-                 ". WSA error: " + std::to_string(WSAGetLastError());
-      ASSERT_TRUE(listen(s, 1) != SOCKET_ERROR)
-          << "Failed to listen on socket " + std::to_string(s) +
-                 ". WSA error: " + std::to_string(WSAGetLastError());
-      sockets.push_back(s);
-    }
-    // Do a non-blocking accept followed by a close on all of those sockets.
-    // Do this in a separate loop to try to induce a time window to hit races.
-    for (size_t i = 0; i < sockets.size(); i++) {
-      ASSERT_TRUE(accept(sockets[i], nullptr, nullptr) == INVALID_SOCKET)
-          << "Accept on phony socket unexpectedly accepted actual connection.";
-      ASSERT_TRUE(WSAGetLastError() == WSAEWOULDBLOCK)
-          << "OpenAndCloseSocketsStressLoop accept on socket " +
-                 std::to_string(sockets[i]) +
-                 " failed in "
-                 "an unexpected way. "
-                 "WSA error: " +
-                 std::to_string(WSAGetLastError()) +
-                 ". Socket use-after-close bugs are likely.";
-      ASSERT_TRUE(closesocket(sockets[i]) != SOCKET_ERROR)
-          << "Failed to close socket: " + std::to_string(sockets[i]) +
-                 ". WSA error: " + std::to_string(WSAGetLastError());
-    }
-  }
+  // TODO(apolcyn): re-enable this on windows if we can debug the failure.
+  // Previously, this was causing test flakes for a while b/c bind calls
+  // would fail with WSAEACCESS. Not clear if we were just making windows
+  // unhappy.
+  LOG(INFO) << "OpenAndCloseSocketsStressLoop is a no-op for windows";
   return;
 }
 #else

test/cpp/end2end/client_lb_end2end_test.cc

@@ -3277,7 +3277,7 @@ TEST_F(WeightedRoundRobinTest, CallAndServerMetric) {
 // all of its subchannels every time it saw an update, thus causing the
 // WRR policy to re-enter the blackout period for that address.
 TEST_F(WeightedRoundRobinTest, WithOutlierDetection) {
-  const int kBlackoutPeriodSeconds = 5;
+  const int kBlackoutPeriodSeconds = 10;
   const int kNumServers = 3;
   StartServers(kNumServers);
   // Report server metrics that should give 6:4:3 WRR picks.

test/cpp/end2end/grpclb_end2end_test.cc

@@ -952,12 +952,19 @@ TEST_F(GrpclbEnd2endTest, UsePickFirstChildPolicy) {
       "}");
   SendBalancerResponse(BuildResponseForBackends(GetBackendPorts(), {}));
   CheckRpcSendOk(kNumRpcs, 3000 /* timeout_ms */, true /* wait_for_ready */);
-  // Check that all requests went to the first backend.  This verifies
-  // that we used pick_first instead of round_robin as the child policy.
-  EXPECT_EQ(backends_[0]->service().request_count(), kNumRpcs);
-  for (size_t i = 1; i < backends_.size(); ++i) {
-    EXPECT_EQ(backends_[i]->service().request_count(), 0UL);
+  // Check that all requests went to one backend.  This verifies that we
+  // used pick_first instead of round_robin as the child policy.
+  bool found = false;
+  for (size_t i = 0; i < backends_.size(); ++i) {
+    if (backends_[i]->service().request_count() > 0) {
+      LOG(INFO) << "backend " << i << " saw traffic";
+      EXPECT_EQ(backends_[i]->service().request_count(), kNumRpcs)
+          << "backend " << i;
+      EXPECT_FALSE(found) << "multiple backends saw traffic";
+      found = true;
+    }
   }
+  EXPECT_TRUE(found) << "no backends saw traffic";
   // The balancer got a single request.
   EXPECT_EQ(1U, balancer_->service().request_count());
   // and sent a single response.
@@ -982,21 +989,24 @@ TEST_F(GrpclbEnd2endTest, SwapChildPolicy) {
       "}");
   SendBalancerResponse(BuildResponseForBackends(GetBackendPorts(), {}));
   CheckRpcSendOk(kNumRpcs, 3000 /* timeout_ms */, true /* wait_for_ready */);
-  // Check that all requests went to the first backend.  This verifies
-  // that we used pick_first instead of round_robin as the child policy.
-  EXPECT_EQ(backends_[0]->service().request_count(), kNumRpcs);
-  for (size_t i = 1; i < backends_.size(); ++i) {
-    EXPECT_EQ(backends_[i]->service().request_count(), 0UL);
+  // Check that all requests went to one backend.  This verifies that we
+  // used pick_first instead of round_robin as the child policy.
+  bool found = false;
+  for (size_t i = 0; i < backends_.size(); ++i) {
+    if (backends_[i]->service().request_count() > 0) {
+      LOG(INFO) << "backend " << i << " saw traffic";
+      EXPECT_EQ(backends_[i]->service().request_count(), kNumRpcs)
+          << "backend " << i;
+      EXPECT_FALSE(found) << "multiple backends saw traffic";
+      found = true;
+    }
   }
+  EXPECT_TRUE(found) << "no backends saw traffic";
   // Send new resolution that removes child policy from service config.
   SetNextResolutionDefaultBalancer();
+  // We should now be using round_robin, which will send traffic to all
+  // backends.
   WaitForAllBackends();
-  CheckRpcSendOk(kNumRpcs, 3000 /* timeout_ms */, true /* wait_for_ready */);
-  // Check that every backend saw the same number of requests.  This verifies
-  // that we used round_robin.
-  for (size_t i = 0; i < backends_.size(); ++i) {
-    EXPECT_EQ(backends_[i]->service().request_count(), 2UL);
-  }
   // The balancer got a single request.
   EXPECT_EQ(1U, balancer_->service().request_count());
   // and sent a single response.

test/cpp/end2end/port_sharing_end2end_test.cc

@@ -38,6 +38,7 @@
 
 #include "src/core/lib/gprpp/crash.h"
 #include "src/core/lib/gprpp/env.h"
+#include "src/core/lib/gprpp/host_port.h"
 #include "src/core/lib/iomgr/endpoint.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
 #include "src/core/lib/iomgr/pollset.h"
@@ -46,6 +47,7 @@
 #include "src/core/lib/security/credentials/credentials.h"
 #include "src/proto/grpc/testing/echo.grpc.pb.h"
 #include "test/core/test_util/port.h"
+#include "test/core/test_util/resolve_localhost_ip46.h"
 #include "test/core/test_util/test_config.h"
 #include "test/core/test_util/test_tcp_server.h"
 #include "test/cpp/end2end/test_service_impl.h"
@@ -96,9 +98,11 @@ class TestTcpServer {
       : shutdown_(false),
         queue_data_(false),
         port_(grpc_pick_unused_port_or_die()) {
-    std::ostringstream server_address;
-    server_address << "localhost:" << port_;
-    address_ = server_address.str();
+    grpc_init();  // needed by LocalIpAndPort()
+    // This test does not do well with multiple connection attempts at the same
+    // time to the same tcp server, so use the local IP address instead of
+    // "localhost" which can result in two connections (ipv4 and ipv6).
+    address_ = grpc_core::LocalIpAndPort(port_);
     test_tcp_server_init(&tcp_server_, &TestTcpServer::OnConnect, this);
     GRPC_CLOSURE_INIT(&on_fd_released_, &TestTcpServer::OnFdReleased, this,
                       grpc_schedule_on_exec_ctx);
@@ -108,6 +112,7 @@ class TestTcpServer {
     running_thread_.join();
     test_tcp_server_destroy(&tcp_server_);
     grpc_recycle_unused_port(port_);
+    grpc_shutdown();
   }
 
   // Read some data before handing off the connection.
@@ -168,7 +173,7 @@ class TestTcpServer {
     grpc_tcp_destroy_and_release_fd(tcp, &fd_, &on_fd_released_);
   }
 
-  void OnFdReleased(grpc_error_handle err) {
+  void OnFdReleased(const absl::Status& err) {
     EXPECT_EQ(absl::OkStatus(), err);
     experimental::ExternalConnectionAcceptor::NewConnectionParameters p;
     p.listener_fd = listener_fd_;

test/cpp/end2end/rls_end2end_test.cc

@@ -234,7 +234,7 @@ class RlsEnd2endTest : public ::testing::Test {
   }
 
   struct RpcOptions {
-    int timeout_ms = 2000;
+    int timeout_ms = 5000;
     bool wait_for_ready = false;
     std::vector<std::pair<std::string, std::string>> metadata;
 
@@ -922,14 +922,15 @@ TEST_F(RlsEnd2endTest, RlsRequestTimeout) {
           .set_default_target(grpc_core::LocalIpUri(backends_[1]->port_))
           .set_lookup_service_timeout(grpc_core::Duration::Seconds(2))
           .Build());
-  // RLS server will send a response, but it's longer than the timeout.
+  // RLS server will send a response, but it takes longer than the
+  // timeout set in the LB policy config.
   rls_server_->service_.SetResponse(
       BuildRlsRequest({{kTestKey, kTestValue}}),
       BuildRlsResponse({grpc_core::LocalIpUri(backends_[0]->port_)}),
       /*response_delay=*/grpc_core::Duration::Seconds(3));
   // The data plane RPC should be sent to the default target.
-  CheckRpcSendOk(DEBUG_LOCATION, RpcOptions().set_timeout_ms(4000).set_metadata(
-                                     {{"key1", kTestValue}}));
+  CheckRpcSendOk(DEBUG_LOCATION,
+                 RpcOptions().set_metadata({{"key1", kTestValue}}));
   EXPECT_EQ(rls_server_->service_.request_count(), 1);
   EXPECT_EQ(backends_[0]->service_.request_count(), 0);
   EXPECT_EQ(backends_[1]->service_.request_count(), 1);

test/cpp/end2end/xds/BUILD

@@ -281,6 +281,7 @@ grpc_cc_test(
     linkstatic = True,  # Fixes dyld error on MacOS
     shard_count = 5,
     tags = [
+        "no_mac",
         "no_test_ios",
         "no_windows",
         "xds_end2end_test",

test/cpp/end2end/xds/xds_core_end2end_test.cc

@@ -1060,7 +1060,7 @@ TEST_P(XdsFederationTest, FederationServer) {
       "xdstp://xds.example.com/envoy.config.listener.v3.Listener"
       "client/%s?client_listener_resource_name_template_not_in_use");
   InitClient(builder);
-  CreateAndStartBackends(2, /*xds_enabled=*/true);
+  CreateBackends(2, /*xds_enabled=*/true);
   // Eds for new authority balancer.
   EdsResourceArgs args =
       EdsResourceArgs({{"locality0", CreateEndpointsForBackends()}});
@@ -1099,6 +1099,13 @@ TEST_P(XdsFederationTest, FederationServer) {
                                      new_server_route_config,
                                      ServerHcmAccessor());
   }
+  // Start backends and wait for them to start serving.
+  StartAllBackends();
+  for (const auto& backend : backends_) {
+    ASSERT_TRUE(backend->notifier()->WaitOnServingStatusChange(
+        grpc_core::LocalIpAndPort(backend->port()), grpc::StatusCode::OK));
+  }
+  // Make sure everything works.
   WaitForAllBackends(DEBUG_LOCATION);
 }
 

test/cpp/end2end/xds/xds_end2end_test_lib.cc

@@ -72,15 +72,26 @@ void XdsEnd2endTest::ServerThread::XdsServingStatusNotifier::
   cond_.Signal();
 }
 
-void XdsEnd2endTest::ServerThread::XdsServingStatusNotifier::
-    WaitOnServingStatusChange(std::string uri,
-                              grpc::StatusCode expected_status) {
+bool XdsEnd2endTest::ServerThread::XdsServingStatusNotifier::
+    WaitOnServingStatusChange(const std::string& uri,
+                              grpc::StatusCode expected_status,
+                              absl::Duration timeout) {
   grpc_core::MutexLock lock(&mu_);
+  absl::Time deadline = absl::Now() + timeout * grpc_test_slowdown_factor();
   std::map<std::string, grpc::Status>::iterator it;
   while ((it = status_map.find(uri)) == status_map.end() ||
          it->second.error_code() != expected_status) {
-    cond_.Wait(&mu_);
+    if (cond_.WaitWithDeadline(&mu_, deadline)) {
+      LOG(ERROR) << "\nTimeout Elapsed waiting on serving status "
+                    "change\nExpected status: "
+                 << expected_status << "\nActual:"
+                 << (it == status_map.end()
+                         ? "Entry not found in map"
+                         : absl::StrCat(it->second.error_code()));
+      return false;
+    }
   }
+  return true;
 }
 
 //
@@ -714,33 +725,37 @@ Status XdsEnd2endTest::LongRunningRpc::GetStatus() {
   return status_;
 }
 
-std::vector<XdsEnd2endTest::ConcurrentRpc> XdsEnd2endTest::SendConcurrentRpcs(
+std::vector<std::unique_ptr<XdsEnd2endTest::ConcurrentRpc>>
+XdsEnd2endTest::SendConcurrentRpcs(
     const grpc_core::DebugLocation& debug_location,
     grpc::testing::EchoTestService::Stub* stub, size_t num_rpcs,
     const RpcOptions& rpc_options) {
   // Variables for RPCs.
-  std::vector<ConcurrentRpc> rpcs(num_rpcs);
+  std::vector<std::unique_ptr<ConcurrentRpc>> rpcs;
+  rpcs.reserve(num_rpcs);
   EchoRequest request;
   // Variables for synchronization
   grpc_core::Mutex mu;
   grpc_core::CondVar cv;
   size_t completed = 0;
   // Set-off callback RPCs
-  for (size_t i = 0; i < num_rpcs; i++) {
-    ConcurrentRpc* rpc = &rpcs[i];
+  for (size_t i = 0; i < num_rpcs; ++i) {
+    auto rpc = std::make_unique<ConcurrentRpc>();
     rpc_options.SetupRpc(&rpc->context, &request);
     grpc_core::Timestamp t0 = NowFromCycleCounter();
-    stub->async()->Echo(&rpc->context, &request, &rpc->response,
-                        [rpc, &mu, &completed, &cv, num_rpcs, t0](Status s) {
-                          rpc->status = s;
-                          rpc->elapsed_time = NowFromCycleCounter() - t0;
-                          bool done;
-                          {
-                            grpc_core::MutexLock lock(&mu);
-                            done = (++completed) == num_rpcs;
-                          }
-                          if (done) cv.Signal();
-                        });
+    stub->async()->Echo(
+        &rpc->context, &request, &rpc->response,
+        [rpc = rpc.get(), &mu, &completed, &cv, num_rpcs, t0](Status s) {
+          rpc->status = s;
+          rpc->elapsed_time = NowFromCycleCounter() - t0;
+          bool done;
+          {
+            grpc_core::MutexLock lock(&mu);
+            done = (++completed) == num_rpcs;
+          }
+          if (done) cv.Signal();
+        });
+    rpcs.push_back(std::move(rpc));
   }
   {
     grpc_core::MutexLock lock(&mu);
@@ -828,11 +843,26 @@ std::string XdsEnd2endTest::MakeConnectionFailureRegex(
       "(Connection refused"
       "|Connection reset by peer"
       "|Socket closed"
+      "|Broken pipe"
       "|FD shutdown)"
       // errno value
       "( \\([0-9]+\\))?");
 }
 
+std::string XdsEnd2endTest::MakeTlsHandshakeFailureRegex(
+    absl::string_view prefix) {
+  return absl::StrCat(
+      prefix,
+      "(UNKNOWN|UNAVAILABLE): "
+      // IP address
+      "(ipv6:%5B::1%5D|ipv4:127.0.0.1):[0-9]+: "
+      // Prefixes added for context
+      "(Failed to connect to remote host: )?"
+      // Tls handshake failure
+      "Tls handshake failed \\(TSI_PROTOCOL_FAILURE\\): SSL_ERROR_SSL: "
+      "error:1000007d:SSL routines:OPENSSL_internal:CERTIFICATE_VERIFY_FAILED");
+}
+
 grpc_core::PemKeyCertPairList XdsEnd2endTest::ReadTlsIdentityPair(
     const char* key_path, const char* cert_path) {
   return grpc_core::PemKeyCertPairList{grpc_core::PemKeyCertPair(

test/cpp/end2end/xds/xds_end2end_test_lib.h

@@ -217,8 +217,9 @@ class XdsEnd2endTest : public ::testing::TestWithParam<XdsTestType>,
       void OnServingStatusUpdate(std::string uri,
                                  ServingStatusUpdate update) override;
 
-      void WaitOnServingStatusChange(std::string uri,
-                                     grpc::StatusCode expected_status);
+      GRPC_MUST_USE_RESULT bool WaitOnServingStatusChange(
+          const std::string& uri, grpc::StatusCode expected_status,
+          absl::Duration timeout = absl::Seconds(10));
 
      private:
       grpc_core::Mutex mu_;
@@ -785,7 +786,7 @@ class XdsEnd2endTest : public ::testing::TestWithParam<XdsTestType>,
     grpc_core::Duration elapsed_time;
     EchoResponse response;
   };
-  std::vector<ConcurrentRpc> SendConcurrentRpcs(
+  std::vector<std::unique_ptr<ConcurrentRpc>> SendConcurrentRpcs(
       const grpc_core::DebugLocation& debug_location,
       grpc::testing::EchoTestService::Stub* stub, size_t num_rpcs,
       const RpcOptions& rpc_options);
@@ -965,6 +966,10 @@ class XdsEnd2endTest : public ::testing::TestWithParam<XdsTestType>,
   // message for a connection failure.
   static std::string MakeConnectionFailureRegex(absl::string_view prefix);
 
+  // Returns a regex that can be matched against an RPC failure status
+  // message for a Tls handshake failure.
+  static std::string MakeTlsHandshakeFailureRegex(absl::string_view prefix);
+
   // Returns a private key pair, read from local files.
   static grpc_core::PemKeyCertPairList ReadTlsIdentityPair(
       const char* key_path, const char* cert_path);

test/cpp/end2end/xds/xds_fault_injection_end2end_test.cc

@@ -240,12 +240,12 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageDelay) {
   // Send kNumRpcs RPCs and count the delays.
   RpcOptions rpc_options =
       RpcOptions().set_timeout(kRpcTimeout).set_skip_cancelled_check(true);
-  std::vector<ConcurrentRpc> rpcs =
+  std::vector<std::unique_ptr<ConcurrentRpc>> rpcs =
       SendConcurrentRpcs(DEBUG_LOCATION, stub_.get(), kNumRpcs, rpc_options);
   size_t num_delayed = 0;
   for (auto& rpc : rpcs) {
-    if (rpc.status.error_code() == StatusCode::OK) continue;
-    EXPECT_EQ(StatusCode::DEADLINE_EXCEEDED, rpc.status.error_code());
+    if (rpc->status.error_code() == StatusCode::OK) continue;
+    EXPECT_EQ(StatusCode::DEADLINE_EXCEEDED, rpc->status.error_code());
     ++num_delayed;
   }
   // The delay rate should be roughly equal to the expectation.
@@ -295,12 +295,12 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageDelayViaHeaders) {
                                .set_metadata(metadata)
                                .set_timeout(kRpcTimeout)
                                .set_skip_cancelled_check(true);
-  std::vector<ConcurrentRpc> rpcs =
+  std::vector<std::unique_ptr<ConcurrentRpc>> rpcs =
       SendConcurrentRpcs(DEBUG_LOCATION, stub_.get(), kNumRpcs, rpc_options);
   size_t num_delayed = 0;
   for (auto& rpc : rpcs) {
-    if (rpc.status.error_code() == StatusCode::OK) continue;
-    EXPECT_EQ(StatusCode::DEADLINE_EXCEEDED, rpc.status.error_code());
+    if (rpc->status.error_code() == StatusCode::OK) continue;
+    EXPECT_EQ(StatusCode::DEADLINE_EXCEEDED, rpc->status.error_code());
     ++num_delayed;
   }
   // The delay rate should be roughly equal to the expectation.
@@ -381,12 +381,12 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionAlwaysDelayPercentageAbort) {
   // Send kNumRpcs RPCs and count the aborts.
   int num_aborted = 0;
   RpcOptions rpc_options = RpcOptions().set_timeout(kRpcTimeout);
-  std::vector<ConcurrentRpc> rpcs =
+  std::vector<std::unique_ptr<ConcurrentRpc>> rpcs =
       SendConcurrentRpcs(DEBUG_LOCATION, stub_.get(), kNumRpcs, rpc_options);
   for (auto& rpc : rpcs) {
-    EXPECT_GE(rpc.elapsed_time, kFixedDelay * grpc_test_slowdown_factor());
-    if (rpc.status.error_code() == StatusCode::OK) continue;
-    EXPECT_EQ("Fault injected", rpc.status.error_message());
+    EXPECT_GE(rpc->elapsed_time, kFixedDelay * grpc_test_slowdown_factor());
+    if (rpc->status.error_code() == StatusCode::OK) continue;
+    EXPECT_EQ("Fault injected", rpc->status.error_message());
     ++num_aborted;
   }
   // The abort rate should be roughly equal to the expectation.
@@ -438,12 +438,12 @@ TEST_P(FaultInjectionTest,
   // Send kNumRpcs RPCs and count the aborts.
   int num_aborted = 0;
   RpcOptions rpc_options = RpcOptions().set_timeout(kRpcTimeout);
-  std::vector<ConcurrentRpc> rpcs =
+  std::vector<std::unique_ptr<ConcurrentRpc>> rpcs =
       SendConcurrentRpcs(DEBUG_LOCATION, stub_.get(), kNumRpcs, rpc_options);
   for (auto& rpc : rpcs) {
-    EXPECT_GE(rpc.elapsed_time, kFixedDelay * grpc_test_slowdown_factor());
-    if (rpc.status.error_code() == StatusCode::OK) continue;
-    EXPECT_EQ("Fault injected", rpc.status.error_message());
+    EXPECT_GE(rpc->elapsed_time, kFixedDelay * grpc_test_slowdown_factor());
+    if (rpc->status.error_code() == StatusCode::OK) continue;
+    EXPECT_EQ("Fault injected", rpc->status.error_message());
     ++num_aborted;
   }
   // The abort rate should be roughly equal to the expectation.
@@ -481,11 +481,11 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionMaxFault) {
   // active faults quota.
   int num_delayed = 0;
   RpcOptions rpc_options = RpcOptions().set_timeout(kRpcTimeout);
-  std::vector<ConcurrentRpc> rpcs =
+  std::vector<std::unique_ptr<ConcurrentRpc>> rpcs =
       SendConcurrentRpcs(DEBUG_LOCATION, stub_.get(), kNumRpcs, rpc_options);
   for (auto& rpc : rpcs) {
-    if (rpc.status.error_code() == StatusCode::OK) continue;
-    EXPECT_EQ(StatusCode::DEADLINE_EXCEEDED, rpc.status.error_code());
+    if (rpc->status.error_code() == StatusCode::OK) continue;
+    EXPECT_EQ(StatusCode::DEADLINE_EXCEEDED, rpc->status.error_code());
     ++num_delayed;
   }
   // Only kMaxFault number of RPC should be fault injected.
@@ -495,8 +495,8 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionMaxFault) {
   num_delayed = 0;
   rpcs = SendConcurrentRpcs(DEBUG_LOCATION, stub_.get(), kNumRpcs, rpc_options);
   for (auto& rpc : rpcs) {
-    if (rpc.status.error_code() == StatusCode::OK) continue;
-    EXPECT_EQ(StatusCode::DEADLINE_EXCEEDED, rpc.status.error_code());
+    if (rpc->status.error_code() == StatusCode::OK) continue;
+    EXPECT_EQ(StatusCode::DEADLINE_EXCEEDED, rpc->status.error_code());
     ++num_delayed;
   }
   // Only kMaxFault number of RPC should be fault injected. If the max fault

tools/distrib/python/grpc_version.py

@@ -14,5 +14,5 @@
 
 # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!!
 
-VERSION = '1.67.0.dev0'
+VERSION = '1.68.0.dev0'
 PROTOBUF_VERSION = '3.27.2'

tools/distrib/python/grpcio_tools/grpc_tools/grpc_version.py

@@ -14,4 +14,4 @@
 
 # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_tools/grpc_version.py.template`!!!
 
-VERSION = '1.67.0.dev0'
+VERSION = '1.68.0.dev0'

tools/distrib/python/grpcio_tools/grpc_version.py

@@ -14,5 +14,5 @@
 
 # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!!
 
-VERSION = '1.67.0.dev0'
+VERSION = '1.68.0.dev0'
 PROTOBUF_VERSION = '3.27.2'

tools/distrib/python/xds_protos/grpc_version.py

@@ -14,5 +14,5 @@
 
 # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!!
 
-VERSION = '1.67.0.dev0'
+VERSION = '1.68.0.dev0'
 PROTOBUF_VERSION = '3.27.2'

tools/doxygen/Doxyfile.c++

@@ -40,7 +40,7 @@ PROJECT_NAME           = "GRPC C++"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = 1.67.0-dev
+PROJECT_NUMBER         = 1.68.0-dev
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

tools/doxygen/Doxyfile.c++.internal

@@ -40,7 +40,7 @@ PROJECT_NAME           = "GRPC C++"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = 1.67.0-dev
+PROJECT_NUMBER         = 1.68.0-dev
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

tools/doxygen/Doxyfile.objc

@@ -40,7 +40,7 @@ PROJECT_NAME           = "GRPC Objective-C"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = 1.67.0-dev
+PROJECT_NUMBER         = 1.68.0-dev
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

tools/doxygen/Doxyfile.objc.internal

@@ -40,7 +40,7 @@ PROJECT_NAME           = "GRPC Objective-C"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = 1.67.0-dev
+PROJECT_NUMBER         = 1.68.0-dev
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

tools/doxygen/Doxyfile.php

@@ -40,7 +40,7 @@ PROJECT_NAME           = "GRPC PHP"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = 1.67.0-dev
+PROJECT_NUMBER         = 1.68.0-dev
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

tools/interop_matrix/client_matrix.py

@@ -305,7 +305,7 @@ LANG_RELEASE_MATRIX = {
             ("v1.63.3", ReleaseInfo()),
             ("v1.64.1", ReleaseInfo()),
             ("v1.65.0", ReleaseInfo()),
-            ("v1.66.0", ReleaseInfo()),
+            ("v1.66.2", ReleaseInfo()),
         ]
     ),
     "java": OrderedDict(