Merge remote-tracking branch 'upstream/master' into gdc_metadata_server

BUILD

@@ -856,6 +856,7 @@ grpc_cc_library(
         "src/core/lib/iomgr/call_combiner.h",
         "src/core/lib/iomgr/closure.h",
         "src/core/lib/iomgr/combiner.h",
+        "src/core/lib/iomgr/dynamic_annotations.h",
         "src/core/lib/iomgr/endpoint.h",
         "src/core/lib/iomgr/endpoint_pair.h",
         "src/core/lib/iomgr/error.h",
@@ -1088,11 +1089,11 @@ grpc_cc_library(
         "grpc_base",
         "grpc_client_authority_filter",
         "grpc_deadline_filter",
+        "health_proto",
         "inlined_vector",
         "orphanable",
         "ref_counted",
         "ref_counted_ptr",
-        "health_proto",
     ],
 )
 
@@ -1590,8 +1591,8 @@ grpc_cc_library(
         "src/core/lib/security/security_connector/load_system_roots_linux.cc",
         "src/core/lib/security/security_connector/local/local_security_connector.cc",
         "src/core/lib/security/security_connector/security_connector.cc",
-        "src/core/lib/security/security_connector/ssl_utils.cc",
         "src/core/lib/security/security_connector/ssl/ssl_security_connector.cc",
+        "src/core/lib/security/security_connector/ssl_utils.cc",
         "src/core/lib/security/transport/client_auth_filter.cc",
         "src/core/lib/security/transport/secure_endpoint.cc",
         "src/core/lib/security/transport/security_handshaker.cc",
@@ -1624,8 +1625,8 @@ grpc_cc_library(
         "src/core/lib/security/security_connector/load_system_roots_linux.h",
         "src/core/lib/security/security_connector/local/local_security_connector.h",
         "src/core/lib/security/security_connector/security_connector.h",
-        "src/core/lib/security/security_connector/ssl_utils.h",
         "src/core/lib/security/security_connector/ssl/ssl_security_connector.h",
+        "src/core/lib/security/security_connector/ssl_utils.h",
         "src/core/lib/security/transport/auth_filters.h",
         "src/core/lib/security/transport/secure_endpoint.h",
         "src/core/lib/security/transport/security_handshaker.h",

build.yaml

@@ -440,6 +440,7 @@ filegroups:
   - src/core/lib/iomgr/call_combiner.h
   - src/core/lib/iomgr/closure.h
   - src/core/lib/iomgr/combiner.h
+  - src/core/lib/iomgr/dynamic_annotations.h
   - src/core/lib/iomgr/endpoint.h
   - src/core/lib/iomgr/endpoint_pair.h
   - src/core/lib/iomgr/error.h

gRPC-C++.podspec

@@ -405,6 +405,7 @@ Pod::Spec.new do |s|
                       'src/core/lib/iomgr/call_combiner.h',
                       'src/core/lib/iomgr/closure.h',
                       'src/core/lib/iomgr/combiner.h',
+                      'src/core/lib/iomgr/dynamic_annotations.h',
                       'src/core/lib/iomgr/endpoint.h',
                       'src/core/lib/iomgr/endpoint_pair.h',
                       'src/core/lib/iomgr/error.h',
@@ -597,6 +598,7 @@ Pod::Spec.new do |s|
                               'src/core/lib/iomgr/call_combiner.h',
                               'src/core/lib/iomgr/closure.h',
                               'src/core/lib/iomgr/combiner.h',
+                              'src/core/lib/iomgr/dynamic_annotations.h',
                               'src/core/lib/iomgr/endpoint.h',
                               'src/core/lib/iomgr/endpoint_pair.h',
                               'src/core/lib/iomgr/error.h',

gRPC-Core.podspec

@@ -403,6 +403,7 @@ Pod::Spec.new do |s|
                       'src/core/lib/iomgr/call_combiner.h',
                       'src/core/lib/iomgr/closure.h',
                       'src/core/lib/iomgr/combiner.h',
+                      'src/core/lib/iomgr/dynamic_annotations.h',
                       'src/core/lib/iomgr/endpoint.h',
                       'src/core/lib/iomgr/endpoint_pair.h',
                       'src/core/lib/iomgr/error.h',
@@ -1022,6 +1023,7 @@ Pod::Spec.new do |s|
                               'src/core/lib/iomgr/call_combiner.h',
                               'src/core/lib/iomgr/closure.h',
                               'src/core/lib/iomgr/combiner.h',
+                              'src/core/lib/iomgr/dynamic_annotations.h',
                               'src/core/lib/iomgr/endpoint.h',
                               'src/core/lib/iomgr/endpoint_pair.h',
                               'src/core/lib/iomgr/error.h',

grpc.gemspec

@@ -339,6 +339,7 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/iomgr/call_combiner.h )
   s.files += %w( src/core/lib/iomgr/closure.h )
   s.files += %w( src/core/lib/iomgr/combiner.h )
+  s.files += %w( src/core/lib/iomgr/dynamic_annotations.h )
   s.files += %w( src/core/lib/iomgr/endpoint.h )
   s.files += %w( src/core/lib/iomgr/endpoint_pair.h )
   s.files += %w( src/core/lib/iomgr/error.h )

include/grpc/impl/codegen/port_platform.h

@@ -526,6 +526,15 @@ typedef unsigned __int64 uint64_t;
 #endif /* GPR_ATTRIBUTE_NO_TSAN (2) */
 #endif /* GPR_ATTRIBUTE_NO_TSAN (1) */
 
+/* GRPC_TSAN_ENABLED will be defined, when compiled with thread sanitizer. */
+#if defined(__SANITIZE_THREAD__)
+#define GRPC_TSAN_ENABLED
+#elif defined(__has_feature)
+#if __has_feature(thread_sanitizer)
+#define GRPC_TSAN_ENABLED
+#endif
+#endif
+
 /* GRPC_ALLOW_EXCEPTIONS should be 0 or 1 if exceptions are allowed or not */
 #ifndef GRPC_ALLOW_EXCEPTIONS
 /* If not already set, set to 1 on Windows (style guide standard) but to

package.xml

@@ -344,6 +344,7 @@
     <file baseinstalldir="/" name="src/core/lib/iomgr/call_combiner.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/closure.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/combiner.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/iomgr/dynamic_annotations.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/endpoint.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/endpoint_pair.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/error.h" role="src" />

src/core/lib/channel/channelz.cc

@@ -208,11 +208,7 @@ char* ServerNode::RenderServerSockets(intptr_t start_socket_id) {
   grpc_json* json = top_level_json;
   grpc_json* json_iterator = nullptr;
   ChildRefsList socket_refs;
-  // uuids index into entities one-off (idx 0 is really uuid 1, since 0 is
-  // reserved). However, we want to support requests coming in with
-  // start_server_id=0, which signifies "give me everything."
-  size_t start_idx = start_socket_id == 0 ? 0 : start_socket_id - 1;
-  grpc_server_populate_server_sockets(server_, &socket_refs, start_idx);
+  grpc_server_populate_server_sockets(server_, &socket_refs, start_socket_id);
   if (!socket_refs.empty()) {
     // create list of socket refs
     grpc_json* array_parent = grpc_json_create_child(

src/core/lib/iomgr/call_combiner.cc

@@ -39,10 +39,57 @@ static gpr_atm encode_cancel_state_error(grpc_error* error) {
   return static_cast<gpr_atm>(1) | (gpr_atm)error;
 }
 
+#ifdef GRPC_TSAN_ENABLED
+static void tsan_closure(void* user_data, grpc_error* error) {
+  grpc_call_combiner* call_combiner =
+      static_cast<grpc_call_combiner*>(user_data);
+  // We ref-count the lock, and check if it's already taken.
+  // If it was taken, we should do nothing. Otherwise, we will mark it as
+  // locked. Note that if two different threads try to do this, only one of
+  // them will be able to mark the lock as acquired, while they both run their
+  // callbacks. In such cases (which should never happen for call_combiner),
+  // TSAN will correctly produce an error.
+  //
+  // TODO(soheil): This only covers the callbacks scheduled by
+  //               grpc_call_combiner_(start|finish). If in the future, a
+  //               callback gets scheduled using other mechanisms, we will need
+  //               to add APIs to externally lock call combiners.
+  grpc_core::RefCountedPtr<grpc_call_combiner::TsanLock> lock =
+      call_combiner->tsan_lock;
+  bool prev = false;
+  if (lock->taken.compare_exchange_strong(prev, true)) {
+    TSAN_ANNOTATE_RWLOCK_ACQUIRED(&lock->taken, true);
+  } else {
+    lock.reset();
+  }
+  GRPC_CLOSURE_RUN(call_combiner->original_closure, GRPC_ERROR_REF(error));
+  if (lock != nullptr) {
+    TSAN_ANNOTATE_RWLOCK_RELEASED(&lock->taken, true);
+    bool prev = true;
+    GPR_ASSERT(lock->taken.compare_exchange_strong(prev, false));
+  }
+}
+#endif
+
+static void call_combiner_sched_closure(grpc_call_combiner* call_combiner,
+                                        grpc_closure* closure,
+                                        grpc_error* error) {
+#ifdef GRPC_TSAN_ENABLED
+  call_combiner->original_closure = closure;
+  GRPC_CLOSURE_SCHED(&call_combiner->tsan_closure, error);
+#else
+  GRPC_CLOSURE_SCHED(closure, error);
+#endif
+}
+
 void grpc_call_combiner_init(grpc_call_combiner* call_combiner) {
   gpr_atm_no_barrier_store(&call_combiner->cancel_state, 0);
   gpr_atm_no_barrier_store(&call_combiner->size, 0);
   gpr_mpscq_init(&call_combiner->queue);
+#ifdef GRPC_TSAN_ENABLED
+  GRPC_CLOSURE_INIT(&call_combiner->tsan_closure, tsan_closure, call_combiner,
+                    grpc_schedule_on_exec_ctx);
+#endif
 }
 
 void grpc_call_combiner_destroy(grpc_call_combiner* call_combiner) {
@@ -87,7 +134,7 @@ void grpc_call_combiner_start(grpc_call_combiner* call_combiner,
       gpr_log(GPR_INFO, "  EXECUTING IMMEDIATELY");
     }
     // Queue was empty, so execute this closure immediately.
-    GRPC_CLOSURE_SCHED(closure, error);
+    call_combiner_sched_closure(call_combiner, closure, error);
   } else {
     if (grpc_call_combiner_trace.enabled()) {
       gpr_log(GPR_INFO, "  QUEUING");
@@ -134,7 +181,8 @@ void grpc_call_combiner_stop(grpc_call_combiner* call_combiner DEBUG_ARGS,
         gpr_log(GPR_INFO, "  EXECUTING FROM QUEUE: closure=%p error=%s",
                 closure, grpc_error_string(closure->error_data.error));
       }
-      GRPC_CLOSURE_SCHED(closure, closure->error_data.error);
+      call_combiner_sched_closure(call_combiner, closure,
+                                  closure->error_data.error);
       break;
     }
   } else if (grpc_call_combiner_trace.enabled()) {

src/core/lib/iomgr/call_combiner.h

@@ -27,7 +27,10 @@
 
 #include "src/core/lib/gpr/mpscq.h"
 #include "src/core/lib/gprpp/inlined_vector.h"
+#include "src/core/lib/gprpp/ref_counted.h"
+#include "src/core/lib/gprpp/ref_counted_ptr.h"
 #include "src/core/lib/iomgr/closure.h"
+#include "src/core/lib/iomgr/dynamic_annotations.h"
 
 // A simple, lock-free mechanism for serializing activity related to a
 // single call.  This is similar to a combiner but is more lightweight.
@@ -40,14 +43,38 @@
 
 extern grpc_core::TraceFlag grpc_call_combiner_trace;
 
-typedef struct {
+struct grpc_call_combiner {
   gpr_atm size = 0;  // size_t, num closures in queue or currently executing
   gpr_mpscq queue;
   // Either 0 (if not cancelled and no cancellation closure set),
   // a grpc_closure* (if the lowest bit is 0),
   // or a grpc_error* (if the lowest bit is 1).
   gpr_atm cancel_state = 0;
-} grpc_call_combiner;
+#ifdef GRPC_TSAN_ENABLED
+  // A fake ref-counted lock that is kept alive after the destruction of
+  // grpc_call_combiner, when we are running the original closure.
+  //
+  // Ideally we want to lock and unlock the call combiner as a pointer, when the
+  // callback is called. However, original_closure is free to trigger
+  // anything on the call combiner (including destruction of grpc_call).
+  // Thus, we need a ref-counted structure that can outlive the call combiner.
+  struct TsanLock
+      : public grpc_core::RefCounted<TsanLock,
+                                     grpc_core::NonPolymorphicRefCount> {
+    TsanLock() { TSAN_ANNOTATE_RWLOCK_CREATE(&taken); }
+    ~TsanLock() { TSAN_ANNOTATE_RWLOCK_DESTROY(&taken); }
+
+    // To avoid double-locking by the same thread, we should acquire/release
+    // the lock only when taken is false. On each acquire taken must be set to
+    // true.
+    std::atomic<bool> taken{false};
+  };
+  grpc_core::RefCountedPtr<TsanLock> tsan_lock =
+      grpc_core::MakeRefCounted<TsanLock>();
+  grpc_closure tsan_closure;
+  grpc_closure* original_closure;
+#endif
+};
 
 // Assumes memory was initialized to zero.
 void grpc_call_combiner_init(grpc_call_combiner* call_combiner);

src/core/lib/iomgr/dynamic_annotations.h

@@ -0,0 +1,67 @@
+/*
+ *
+ * Copyright 2018 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_IOMGR_DYNAMIC_ANNOTATIONS_H
+#define GRPC_CORE_LIB_IOMGR_DYNAMIC_ANNOTATIONS_H
+
+#include <grpc/support/port_platform.h>
+
+#ifdef GRPC_TSAN_ENABLED
+
+#define TSAN_ANNOTATE_HAPPENS_BEFORE(addr) \
+  AnnotateHappensBefore(__FILE__, __LINE__, (void*)(addr))
+#define TSAN_ANNOTATE_HAPPENS_AFTER(addr) \
+  AnnotateHappensAfter(__FILE__, __LINE__, (void*)(addr))
+#define TSAN_ANNOTATE_RWLOCK_CREATE(addr) \
+  AnnotateRWLockCreate(__FILE__, __LINE__, (void*)(addr))
+#define TSAN_ANNOTATE_RWLOCK_DESTROY(addr) \
+  AnnotateRWLockDestroy(__FILE__, __LINE__, (void*)(addr))
+#define TSAN_ANNOTATE_RWLOCK_ACQUIRED(addr, is_w) \
+  AnnotateRWLockAcquired(__FILE__, __LINE__, (void*)(addr), (is_w))
+#define TSAN_ANNOTATE_RWLOCK_RELEASED(addr, is_w) \
+  AnnotateRWLockReleased(__FILE__, __LINE__, (void*)(addr), (is_w))
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+void AnnotateHappensBefore(const char* file, int line, const volatile void* cv);
+void AnnotateHappensAfter(const char* file, int line, const volatile void* cv);
+void AnnotateRWLockCreate(const char* file, int line,
+                          const volatile void* lock);
+void AnnotateRWLockDestroy(const char* file, int line,
+                           const volatile void* lock);
+void AnnotateRWLockAcquired(const char* file, int line,
+                            const volatile void* lock, long is_w);
+void AnnotateRWLockReleased(const char* file, int line,
+                            const volatile void* lock, long is_w);
+#ifdef __cplusplus
+}
+#endif
+
+#else /* GRPC_TSAN_ENABLED */
+
+#define TSAN_ANNOTATE_HAPPENS_BEFORE(addr)
+#define TSAN_ANNOTATE_HAPPENS_AFTER(addr)
+#define TSAN_ANNOTATE_RWLOCK_CREATE(addr)
+#define TSAN_ANNOTATE_RWLOCK_DESTROY(addr)
+#define TSAN_ANNOTATE_RWLOCK_ACQUIRED(addr, is_w)
+#define TSAN_ANNOTATE_RWLOCK_RELEASED(addr, is_w)
+
+#endif /* GRPC_TSAN_ENABLED */
+
+#endif /* GRPC_CORE_LIB_IOMGR_DYNAMIC_ANNOTATIONS_H */

test/cpp/microbenchmarks/BUILD

@@ -29,7 +29,6 @@ grpc_cc_test(
 
 grpc_cc_library(
     name = "helpers",
-    testonly = 1,
     srcs = ["helpers.cc"],
     hdrs = [
         "fullstack_context_mutators.h",
@@ -47,70 +46,64 @@ grpc_cc_library(
     ],
 )
 
-grpc_cc_test(
+grpc_cc_binary(
     name = "bm_closure",
+    testonly = 1,
     srcs = ["bm_closure.cc"],
     deps = [":helpers"],
-    uses_polling = False,
 )
 
-# TODO(https://github.com/grpc/grpc/pull/16882): make this a test target
-# right now it OOMs
 grpc_cc_binary(
     name = "bm_arena",
-    testonly = 1,
     srcs = ["bm_arena.cc"],
     deps = [":helpers"],
 )
 
-grpc_cc_test(
+grpc_cc_binary(
     name = "bm_channel",
+    testonly = 1,
     srcs = ["bm_channel.cc"],
     deps = [":helpers"],
-    uses_polling = False,
 )
 
-# TODO(https://github.com/grpc/grpc/pull/16882): make this a test target
-# right now it fails UBSAN
 grpc_cc_binary(
     name = "bm_call_create",
-    testonly = 1,
     srcs = ["bm_call_create.cc"],
     deps = [":helpers"],
 )
 
-grpc_cc_test(
+grpc_cc_binary(
     name = "bm_cq",
+    testonly = 1,
     srcs = ["bm_cq.cc"],
     deps = [":helpers"],
-    uses_polling = False,
 )
 
-grpc_cc_test(
+grpc_cc_binary(
     name = "bm_cq_multiple_threads",
+    testonly = 1,
     srcs = ["bm_cq_multiple_threads.cc"],
     deps = [":helpers"],
-    uses_polling = False,
 )
 
-grpc_cc_test(
+grpc_cc_binary(
     name = "bm_error",
+    testonly = 1,
     srcs = ["bm_error.cc"],
     deps = [":helpers"],
-    uses_polling = False,
 )
 
 grpc_cc_library(
     name = "fullstack_streaming_ping_pong_h",
-    testonly = 1,
     hdrs = [
         "fullstack_streaming_ping_pong.h",
     ],
     deps = [":helpers"],
 )
 
-grpc_cc_test(
+grpc_cc_binary(
     name = "bm_fullstack_streaming_ping_pong",
+    testonly = 1,
     srcs = [
         "bm_fullstack_streaming_ping_pong.cc",
     ],
@@ -119,15 +112,15 @@ grpc_cc_test(
 
 grpc_cc_library(
     name = "fullstack_streaming_pump_h",
-    testonly = 1,
     hdrs = [
         "fullstack_streaming_pump.h",
     ],
     deps = [":helpers"],
 )
 
-grpc_cc_test(
+grpc_cc_binary(
     name = "bm_fullstack_streaming_pump",
+    testonly = 1,
     srcs = [
         "bm_fullstack_streaming_pump.cc",
     ],
@@ -136,52 +129,39 @@ grpc_cc_test(
 
 grpc_cc_binary(
     name = "bm_fullstack_trickle",
-    testonly = 1,
     srcs = ["bm_fullstack_trickle.cc"],
     deps = [":helpers"],
 )
 
 grpc_cc_library(
     name = "fullstack_unary_ping_pong_h",
-    testonly = 1,
     hdrs = [
         "fullstack_unary_ping_pong.h",
     ],
     deps = [":helpers"],
 )
 
-grpc_cc_test(
+grpc_cc_binary(
     name = "bm_fullstack_unary_ping_pong",
+    testonly = 1,
     srcs = [
         "bm_fullstack_unary_ping_pong.cc",
     ],
     deps = [":fullstack_unary_ping_pong_h"],
 )
 
-grpc_cc_test(
+grpc_cc_binary(
     name = "bm_metadata",
+    testonly = 1,
     srcs = ["bm_metadata.cc"],
     deps = [":helpers"],
-    uses_polling = False,
 )
 
-grpc_cc_test(
+grpc_cc_binary(
     name = "bm_chttp2_hpack",
+    testonly = 1,
     srcs = ["bm_chttp2_hpack.cc"],
     deps = [":helpers"],
-    uses_polling = False,
-)
-
-grpc_cc_test(
-    name = "bm_chttp2_transport",
-    srcs = ["bm_chttp2_transport.cc"],
-    deps = [":helpers"],
-)
-
-grpc_cc_test(
-    name = "bm_pollset",
-    srcs = ["bm_pollset.cc"],
-    deps = [":helpers"],
 )
 
 grpc_cc_binary(

test/cpp/microbenchmarks/bm_call_create.cc

@@ -526,7 +526,7 @@ static void BM_IsolatedFilter(benchmark::State& state) {
 
   grpc_core::ExecCtx exec_ctx;
   size_t channel_size = grpc_channel_stack_size(
-      filters.size() == 0 ? nullptr : filters.data(), filters.size());
+      filters.size() == 0 ? nullptr : &filters[0], filters.size());
   grpc_channel_stack* channel_stack =
       static_cast<grpc_channel_stack*>(gpr_zalloc(channel_size));
   GPR_ASSERT(GRPC_LOG_IF_ERROR(

tools/doxygen/Doxyfile.c++.internal

@@ -1084,6 +1084,7 @@ src/core/lib/iomgr/buffer_list.h \
 src/core/lib/iomgr/call_combiner.h \
 src/core/lib/iomgr/closure.h \
 src/core/lib/iomgr/combiner.h \
+src/core/lib/iomgr/dynamic_annotations.h \
 src/core/lib/iomgr/endpoint.h \
 src/core/lib/iomgr/endpoint_pair.h \
 src/core/lib/iomgr/error.h \

tools/doxygen/Doxyfile.core.internal

@@ -1182,6 +1182,7 @@ src/core/lib/iomgr/call_combiner.h \
 src/core/lib/iomgr/closure.h \
 src/core/lib/iomgr/combiner.cc \
 src/core/lib/iomgr/combiner.h \
+src/core/lib/iomgr/dynamic_annotations.h \
 src/core/lib/iomgr/endpoint.cc \
 src/core/lib/iomgr/endpoint.h \
 src/core/lib/iomgr/endpoint_pair.h \

tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh

@@ -15,8 +15,8 @@
 
 set -ex
 
-# A temporary solution to give Kokoro credentials. 
-# The file name 4321_grpc-testing-service needs to match auth_credential in 
+# A temporary solution to give Kokoro credentials.
+# The file name 4321_grpc-testing-service needs to match auth_credential in
 # the build config.
 mkdir -p ${KOKORO_KEYSTORE_DIR}
 cp ${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json ${KOKORO_KEYSTORE_DIR}/4321_grpc-testing-service
@@ -35,14 +35,15 @@ cd $(dirname $0)/../../..
 source tools/internal_ci/helper_scripts/prepare_build_linux_rc
 
 # to get "bazel" link for kokoro build, we need to generate
-# invocation UUID, set an env var for bazel to pick it up
-#  and upload "bazel_invocation_ids" file as artifact.
-export BAZEL_INTERNAL_INVOCATION_ID="$(uuidgen)"
-echo "${BAZEL_INTERNAL_INVOCATION_ID}" >"${KOKORO_ARTIFACTS_DIR}/bazel_invocation_ids"
+# invocation UUID, set a flag for bazel to use it
+# and upload "bazel_invocation_ids" file as artifact.
+BAZEL_INVOCATION_ID="$(uuidgen)"
+echo "${BAZEL_INVOCATION_ID}" >"${KOKORO_ARTIFACTS_DIR}/bazel_invocation_ids"
 
 bazel \
   --bazelrc=tools/remote_build/kokoro.bazelrc \
   test \
+  --invocation_id="${BAZEL_INVOCATION_ID}" \
   $@ \
   -- //test/... || FAILED="true"
 

tools/remote_build/kokoro.bazelrc

@@ -26,7 +26,6 @@ build --auth_credentials=/tmpfs/src/keystore/4321_grpc-testing-service
 build --auth_scope=https://www.googleapis.com/auth/cloud-source-tools
 
 build --bes_backend=buildeventservice.googleapis.com
-build --bes_best_effort=false
 build --bes_timeout=600s
 build --project_id=grpc-testing
 

tools/run_tests/artifacts/build_package_python.sh

@@ -19,20 +19,10 @@ cd "$(dirname "$0")/../../.."
 
 mkdir -p artifacts/
 
+# All the python packages have been built in the artifact phase already
+# and we only collect them here to deliver them to the distribtest phase.
 cp -r "${EXTERNAL_GIT_ROOT}"/input_artifacts/python_*/* artifacts/ || true
 
-strip_binary_wheel() {
-  TEMP_WHEEL_DIR=$(mktemp -d)
-  unzip "$1" -d "$TEMP_WHEEL_DIR"
-  find "$TEMP_WHEEL_DIR" -name "_protoc_compiler*.so" -exec strip --strip-debug {} ";"
-  find "$TEMP_WHEEL_DIR" -name "cygrpc*.so" -exec strip --strip-debug {} ";"
-  (cd "$TEMP_WHEEL_DIR" && zip -r - .) > "$1"
-}
-
-for wheel in artifacts/*.whl; do
-    strip_binary_wheel "$wheel"
-done
-
 # TODO: all the artifact builder configurations generate a grpcio-VERSION.tar.gz
 # source distribution package, and only one of them will end up
 # in the artifacts/ directory. They should be all equivalent though.

tools/run_tests/generated/sources_and_headers.json

@@ -9738,6 +9738,7 @@
       "src/core/lib/iomgr/call_combiner.h", 
       "src/core/lib/iomgr/closure.h", 
       "src/core/lib/iomgr/combiner.h", 
+      "src/core/lib/iomgr/dynamic_annotations.h", 
       "src/core/lib/iomgr/endpoint.h", 
       "src/core/lib/iomgr/endpoint_pair.h", 
       "src/core/lib/iomgr/error.h", 
@@ -9890,6 +9891,7 @@
       "src/core/lib/iomgr/call_combiner.h", 
       "src/core/lib/iomgr/closure.h", 
       "src/core/lib/iomgr/combiner.h", 
+      "src/core/lib/iomgr/dynamic_annotations.h", 
       "src/core/lib/iomgr/endpoint.h", 
       "src/core/lib/iomgr/endpoint_pair.h", 
       "src/core/lib/iomgr/error.h",