diff --git a/BUILD b/BUILD index 9a2c16c6012..9e3e594038d 100644 --- a/BUILD +++ b/BUILD @@ -856,6 +856,7 @@ grpc_cc_library( "src/core/lib/iomgr/call_combiner.h", "src/core/lib/iomgr/closure.h", "src/core/lib/iomgr/combiner.h", + "src/core/lib/iomgr/dynamic_annotations.h", "src/core/lib/iomgr/endpoint.h", "src/core/lib/iomgr/endpoint_pair.h", "src/core/lib/iomgr/error.h", @@ -1088,11 +1089,11 @@ grpc_cc_library( "grpc_base", "grpc_client_authority_filter", "grpc_deadline_filter", + "health_proto", "inlined_vector", "orphanable", "ref_counted", "ref_counted_ptr", - "health_proto", ], ) @@ -1590,8 +1591,8 @@ grpc_cc_library( "src/core/lib/security/security_connector/load_system_roots_linux.cc", "src/core/lib/security/security_connector/local/local_security_connector.cc", "src/core/lib/security/security_connector/security_connector.cc", - "src/core/lib/security/security_connector/ssl_utils.cc", "src/core/lib/security/security_connector/ssl/ssl_security_connector.cc", + "src/core/lib/security/security_connector/ssl_utils.cc", "src/core/lib/security/transport/client_auth_filter.cc", "src/core/lib/security/transport/secure_endpoint.cc", "src/core/lib/security/transport/security_handshaker.cc", @@ -1624,8 +1625,8 @@ grpc_cc_library( "src/core/lib/security/security_connector/load_system_roots_linux.h", "src/core/lib/security/security_connector/local/local_security_connector.h", "src/core/lib/security/security_connector/security_connector.h", - "src/core/lib/security/security_connector/ssl_utils.h", "src/core/lib/security/security_connector/ssl/ssl_security_connector.h", + "src/core/lib/security/security_connector/ssl_utils.h", "src/core/lib/security/transport/auth_filters.h", "src/core/lib/security/transport/secure_endpoint.h", "src/core/lib/security/transport/security_handshaker.h", diff --git a/build.yaml b/build.yaml index 41c63d133a9..1e63933f555 100644 --- a/build.yaml +++ b/build.yaml @@ -440,6 +440,7 @@ filegroups: - src/core/lib/iomgr/call_combiner.h - src/core/lib/iomgr/closure.h - src/core/lib/iomgr/combiner.h + - src/core/lib/iomgr/dynamic_annotations.h - src/core/lib/iomgr/endpoint.h - src/core/lib/iomgr/endpoint_pair.h - src/core/lib/iomgr/error.h diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index 6647201714c..5e79803497f 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -405,6 +405,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/call_combiner.h', 'src/core/lib/iomgr/closure.h', 'src/core/lib/iomgr/combiner.h', + 'src/core/lib/iomgr/dynamic_annotations.h', 'src/core/lib/iomgr/endpoint.h', 'src/core/lib/iomgr/endpoint_pair.h', 'src/core/lib/iomgr/error.h', @@ -597,6 +598,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/call_combiner.h', 'src/core/lib/iomgr/closure.h', 'src/core/lib/iomgr/combiner.h', + 'src/core/lib/iomgr/dynamic_annotations.h', 'src/core/lib/iomgr/endpoint.h', 'src/core/lib/iomgr/endpoint_pair.h', 'src/core/lib/iomgr/error.h', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 3ec0852ad31..1d4e1ae35c0 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -403,6 +403,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/call_combiner.h', 'src/core/lib/iomgr/closure.h', 'src/core/lib/iomgr/combiner.h', + 'src/core/lib/iomgr/dynamic_annotations.h', 'src/core/lib/iomgr/endpoint.h', 'src/core/lib/iomgr/endpoint_pair.h', 'src/core/lib/iomgr/error.h', @@ -1022,6 +1023,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/call_combiner.h', 'src/core/lib/iomgr/closure.h', 'src/core/lib/iomgr/combiner.h', + 'src/core/lib/iomgr/dynamic_annotations.h', 'src/core/lib/iomgr/endpoint.h', 'src/core/lib/iomgr/endpoint_pair.h', 'src/core/lib/iomgr/error.h', diff --git a/grpc.gemspec b/grpc.gemspec index b5a1ef43fd3..92b1e0be689 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -339,6 +339,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/call_combiner.h ) s.files += %w( src/core/lib/iomgr/closure.h ) s.files += %w( src/core/lib/iomgr/combiner.h ) + s.files += %w( src/core/lib/iomgr/dynamic_annotations.h ) s.files += %w( src/core/lib/iomgr/endpoint.h ) s.files += %w( src/core/lib/iomgr/endpoint_pair.h ) s.files += %w( src/core/lib/iomgr/error.h ) diff --git a/include/grpc/impl/codegen/port_platform.h b/include/grpc/impl/codegen/port_platform.h index b2028a63053..031c0c36aef 100644 --- a/include/grpc/impl/codegen/port_platform.h +++ b/include/grpc/impl/codegen/port_platform.h @@ -526,6 +526,15 @@ typedef unsigned __int64 uint64_t; #endif /* GPR_ATTRIBUTE_NO_TSAN (2) */ #endif /* GPR_ATTRIBUTE_NO_TSAN (1) */ +/* GRPC_TSAN_ENABLED will be defined, when compiled with thread sanitizer. */ +#if defined(__SANITIZE_THREAD__) +#define GRPC_TSAN_ENABLED +#elif defined(__has_feature) +#if __has_feature(thread_sanitizer) +#define GRPC_TSAN_ENABLED +#endif +#endif + /* GRPC_ALLOW_EXCEPTIONS should be 0 or 1 if exceptions are allowed or not */ #ifndef GRPC_ALLOW_EXCEPTIONS /* If not already set, set to 1 on Windows (style guide standard) but to diff --git a/package.xml b/package.xml index a354ad5fa7d..bdcb12bfc5e 100644 --- a/package.xml +++ b/package.xml @@ -344,6 +344,7 @@ + diff --git a/src/core/lib/channel/channelz.cc b/src/core/lib/channel/channelz.cc index 8d589f5983e..0802143fbeb 100644 --- a/src/core/lib/channel/channelz.cc +++ b/src/core/lib/channel/channelz.cc @@ -208,11 +208,7 @@ char* ServerNode::RenderServerSockets(intptr_t start_socket_id) { grpc_json* json = top_level_json; grpc_json* json_iterator = nullptr; ChildRefsList socket_refs; - // uuids index into entities one-off (idx 0 is really uuid 1, since 0 is - // reserved). However, we want to support requests coming in with - // start_server_id=0, which signifies "give me everything." - size_t start_idx = start_socket_id == 0 ? 0 : start_socket_id - 1; - grpc_server_populate_server_sockets(server_, &socket_refs, start_idx); + grpc_server_populate_server_sockets(server_, &socket_refs, start_socket_id); if (!socket_refs.empty()) { // create list of socket refs grpc_json* array_parent = grpc_json_create_child( diff --git a/src/core/lib/iomgr/call_combiner.cc b/src/core/lib/iomgr/call_combiner.cc index 90dda45ba37..6b5759a036f 100644 --- a/src/core/lib/iomgr/call_combiner.cc +++ b/src/core/lib/iomgr/call_combiner.cc @@ -39,10 +39,57 @@ static gpr_atm encode_cancel_state_error(grpc_error* error) { return static_cast(1) | (gpr_atm)error; } +#ifdef GRPC_TSAN_ENABLED +static void tsan_closure(void* user_data, grpc_error* error) { + grpc_call_combiner* call_combiner = + static_cast(user_data); + // We ref-count the lock, and check if it's already taken. + // If it was taken, we should do nothing. Otherwise, we will mark it as + // locked. Note that if two different threads try to do this, only one of + // them will be able to mark the lock as acquired, while they both run their + // callbacks. In such cases (which should never happen for call_combiner), + // TSAN will correctly produce an error. + // + // TODO(soheil): This only covers the callbacks scheduled by + // grpc_call_combiner_(start|finish). If in the future, a + // callback gets scheduled using other mechanisms, we will need + // to add APIs to externally lock call combiners. + grpc_core::RefCountedPtr lock = + call_combiner->tsan_lock; + bool prev = false; + if (lock->taken.compare_exchange_strong(prev, true)) { + TSAN_ANNOTATE_RWLOCK_ACQUIRED(&lock->taken, true); + } else { + lock.reset(); + } + GRPC_CLOSURE_RUN(call_combiner->original_closure, GRPC_ERROR_REF(error)); + if (lock != nullptr) { + TSAN_ANNOTATE_RWLOCK_RELEASED(&lock->taken, true); + bool prev = true; + GPR_ASSERT(lock->taken.compare_exchange_strong(prev, false)); + } +} +#endif + +static void call_combiner_sched_closure(grpc_call_combiner* call_combiner, + grpc_closure* closure, + grpc_error* error) { +#ifdef GRPC_TSAN_ENABLED + call_combiner->original_closure = closure; + GRPC_CLOSURE_SCHED(&call_combiner->tsan_closure, error); +#else + GRPC_CLOSURE_SCHED(closure, error); +#endif +} + void grpc_call_combiner_init(grpc_call_combiner* call_combiner) { gpr_atm_no_barrier_store(&call_combiner->cancel_state, 0); gpr_atm_no_barrier_store(&call_combiner->size, 0); gpr_mpscq_init(&call_combiner->queue); +#ifdef GRPC_TSAN_ENABLED + GRPC_CLOSURE_INIT(&call_combiner->tsan_closure, tsan_closure, call_combiner, + grpc_schedule_on_exec_ctx); +#endif } void grpc_call_combiner_destroy(grpc_call_combiner* call_combiner) { @@ -87,7 +134,7 @@ void grpc_call_combiner_start(grpc_call_combiner* call_combiner, gpr_log(GPR_INFO, " EXECUTING IMMEDIATELY"); } // Queue was empty, so execute this closure immediately. - GRPC_CLOSURE_SCHED(closure, error); + call_combiner_sched_closure(call_combiner, closure, error); } else { if (grpc_call_combiner_trace.enabled()) { gpr_log(GPR_INFO, " QUEUING"); @@ -134,7 +181,8 @@ void grpc_call_combiner_stop(grpc_call_combiner* call_combiner DEBUG_ARGS, gpr_log(GPR_INFO, " EXECUTING FROM QUEUE: closure=%p error=%s", closure, grpc_error_string(closure->error_data.error)); } - GRPC_CLOSURE_SCHED(closure, closure->error_data.error); + call_combiner_sched_closure(call_combiner, closure, + closure->error_data.error); break; } } else if (grpc_call_combiner_trace.enabled()) { diff --git a/src/core/lib/iomgr/call_combiner.h b/src/core/lib/iomgr/call_combiner.h index c943fb15570..4ec0044f056 100644 --- a/src/core/lib/iomgr/call_combiner.h +++ b/src/core/lib/iomgr/call_combiner.h @@ -27,7 +27,10 @@ #include "src/core/lib/gpr/mpscq.h" #include "src/core/lib/gprpp/inlined_vector.h" +#include "src/core/lib/gprpp/ref_counted.h" +#include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/iomgr/closure.h" +#include "src/core/lib/iomgr/dynamic_annotations.h" // A simple, lock-free mechanism for serializing activity related to a // single call. This is similar to a combiner but is more lightweight. @@ -40,14 +43,38 @@ extern grpc_core::TraceFlag grpc_call_combiner_trace; -typedef struct { +struct grpc_call_combiner { gpr_atm size = 0; // size_t, num closures in queue or currently executing gpr_mpscq queue; // Either 0 (if not cancelled and no cancellation closure set), // a grpc_closure* (if the lowest bit is 0), // or a grpc_error* (if the lowest bit is 1). gpr_atm cancel_state = 0; -} grpc_call_combiner; +#ifdef GRPC_TSAN_ENABLED + // A fake ref-counted lock that is kept alive after the destruction of + // grpc_call_combiner, when we are running the original closure. + // + // Ideally we want to lock and unlock the call combiner as a pointer, when the + // callback is called. However, original_closure is free to trigger + // anything on the call combiner (including destruction of grpc_call). + // Thus, we need a ref-counted structure that can outlive the call combiner. + struct TsanLock + : public grpc_core::RefCounted { + TsanLock() { TSAN_ANNOTATE_RWLOCK_CREATE(&taken); } + ~TsanLock() { TSAN_ANNOTATE_RWLOCK_DESTROY(&taken); } + + // To avoid double-locking by the same thread, we should acquire/release + // the lock only when taken is false. On each acquire taken must be set to + // true. + std::atomic taken{false}; + }; + grpc_core::RefCountedPtr tsan_lock = + grpc_core::MakeRefCounted(); + grpc_closure tsan_closure; + grpc_closure* original_closure; +#endif +}; // Assumes memory was initialized to zero. void grpc_call_combiner_init(grpc_call_combiner* call_combiner); diff --git a/src/core/lib/iomgr/dynamic_annotations.h b/src/core/lib/iomgr/dynamic_annotations.h new file mode 100644 index 00000000000..713928023ae --- /dev/null +++ b/src/core/lib/iomgr/dynamic_annotations.h @@ -0,0 +1,67 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPC_CORE_LIB_IOMGR_DYNAMIC_ANNOTATIONS_H +#define GRPC_CORE_LIB_IOMGR_DYNAMIC_ANNOTATIONS_H + +#include + +#ifdef GRPC_TSAN_ENABLED + +#define TSAN_ANNOTATE_HAPPENS_BEFORE(addr) \ + AnnotateHappensBefore(__FILE__, __LINE__, (void*)(addr)) +#define TSAN_ANNOTATE_HAPPENS_AFTER(addr) \ + AnnotateHappensAfter(__FILE__, __LINE__, (void*)(addr)) +#define TSAN_ANNOTATE_RWLOCK_CREATE(addr) \ + AnnotateRWLockCreate(__FILE__, __LINE__, (void*)(addr)) +#define TSAN_ANNOTATE_RWLOCK_DESTROY(addr) \ + AnnotateRWLockDestroy(__FILE__, __LINE__, (void*)(addr)) +#define TSAN_ANNOTATE_RWLOCK_ACQUIRED(addr, is_w) \ + AnnotateRWLockAcquired(__FILE__, __LINE__, (void*)(addr), (is_w)) +#define TSAN_ANNOTATE_RWLOCK_RELEASED(addr, is_w) \ + AnnotateRWLockReleased(__FILE__, __LINE__, (void*)(addr), (is_w)) + +#ifdef __cplusplus +extern "C" { +#endif +void AnnotateHappensBefore(const char* file, int line, const volatile void* cv); +void AnnotateHappensAfter(const char* file, int line, const volatile void* cv); +void AnnotateRWLockCreate(const char* file, int line, + const volatile void* lock); +void AnnotateRWLockDestroy(const char* file, int line, + const volatile void* lock); +void AnnotateRWLockAcquired(const char* file, int line, + const volatile void* lock, long is_w); +void AnnotateRWLockReleased(const char* file, int line, + const volatile void* lock, long is_w); +#ifdef __cplusplus +} +#endif + +#else /* GRPC_TSAN_ENABLED */ + +#define TSAN_ANNOTATE_HAPPENS_BEFORE(addr) +#define TSAN_ANNOTATE_HAPPENS_AFTER(addr) +#define TSAN_ANNOTATE_RWLOCK_CREATE(addr) +#define TSAN_ANNOTATE_RWLOCK_DESTROY(addr) +#define TSAN_ANNOTATE_RWLOCK_ACQUIRED(addr, is_w) +#define TSAN_ANNOTATE_RWLOCK_RELEASED(addr, is_w) + +#endif /* GRPC_TSAN_ENABLED */ + +#endif /* GRPC_CORE_LIB_IOMGR_DYNAMIC_ANNOTATIONS_H */ diff --git a/test/cpp/microbenchmarks/BUILD b/test/cpp/microbenchmarks/BUILD index afac414e09d..5ae9a9a7910 100644 --- a/test/cpp/microbenchmarks/BUILD +++ b/test/cpp/microbenchmarks/BUILD @@ -29,7 +29,6 @@ grpc_cc_test( grpc_cc_library( name = "helpers", - testonly = 1, srcs = ["helpers.cc"], hdrs = [ "fullstack_context_mutators.h", @@ -47,70 +46,64 @@ grpc_cc_library( ], ) -grpc_cc_test( +grpc_cc_binary( name = "bm_closure", + testonly = 1, srcs = ["bm_closure.cc"], deps = [":helpers"], - uses_polling = False, ) -# TODO(https://github.com/grpc/grpc/pull/16882): make this a test target -# right now it OOMs grpc_cc_binary( name = "bm_arena", - testonly = 1, srcs = ["bm_arena.cc"], deps = [":helpers"], ) -grpc_cc_test( +grpc_cc_binary( name = "bm_channel", + testonly = 1, srcs = ["bm_channel.cc"], deps = [":helpers"], - uses_polling = False, ) -# TODO(https://github.com/grpc/grpc/pull/16882): make this a test target -# right now it fails UBSAN grpc_cc_binary( name = "bm_call_create", - testonly = 1, srcs = ["bm_call_create.cc"], deps = [":helpers"], ) -grpc_cc_test( +grpc_cc_binary( name = "bm_cq", + testonly = 1, srcs = ["bm_cq.cc"], deps = [":helpers"], - uses_polling = False, ) -grpc_cc_test( +grpc_cc_binary( name = "bm_cq_multiple_threads", + testonly = 1, srcs = ["bm_cq_multiple_threads.cc"], deps = [":helpers"], - uses_polling = False, ) -grpc_cc_test( +grpc_cc_binary( name = "bm_error", + testonly = 1, srcs = ["bm_error.cc"], deps = [":helpers"], - uses_polling = False, ) grpc_cc_library( name = "fullstack_streaming_ping_pong_h", - testonly = 1, hdrs = [ "fullstack_streaming_ping_pong.h", ], deps = [":helpers"], ) -grpc_cc_test( +grpc_cc_binary( name = "bm_fullstack_streaming_ping_pong", + testonly = 1, srcs = [ "bm_fullstack_streaming_ping_pong.cc", ], @@ -119,15 +112,15 @@ grpc_cc_test( grpc_cc_library( name = "fullstack_streaming_pump_h", - testonly = 1, hdrs = [ "fullstack_streaming_pump.h", ], deps = [":helpers"], ) -grpc_cc_test( +grpc_cc_binary( name = "bm_fullstack_streaming_pump", + testonly = 1, srcs = [ "bm_fullstack_streaming_pump.cc", ], @@ -136,52 +129,39 @@ grpc_cc_test( grpc_cc_binary( name = "bm_fullstack_trickle", - testonly = 1, srcs = ["bm_fullstack_trickle.cc"], deps = [":helpers"], ) grpc_cc_library( name = "fullstack_unary_ping_pong_h", - testonly = 1, hdrs = [ "fullstack_unary_ping_pong.h", ], deps = [":helpers"], ) -grpc_cc_test( +grpc_cc_binary( name = "bm_fullstack_unary_ping_pong", + testonly = 1, srcs = [ "bm_fullstack_unary_ping_pong.cc", ], deps = [":fullstack_unary_ping_pong_h"], ) -grpc_cc_test( +grpc_cc_binary( name = "bm_metadata", + testonly = 1, srcs = ["bm_metadata.cc"], deps = [":helpers"], - uses_polling = False, ) -grpc_cc_test( +grpc_cc_binary( name = "bm_chttp2_hpack", + testonly = 1, srcs = ["bm_chttp2_hpack.cc"], deps = [":helpers"], - uses_polling = False, -) - -grpc_cc_test( - name = "bm_chttp2_transport", - srcs = ["bm_chttp2_transport.cc"], - deps = [":helpers"], -) - -grpc_cc_test( - name = "bm_pollset", - srcs = ["bm_pollset.cc"], - deps = [":helpers"], ) grpc_cc_binary( diff --git a/test/cpp/microbenchmarks/bm_call_create.cc b/test/cpp/microbenchmarks/bm_call_create.cc index a0157c66c25..8d12606434b 100644 --- a/test/cpp/microbenchmarks/bm_call_create.cc +++ b/test/cpp/microbenchmarks/bm_call_create.cc @@ -526,7 +526,7 @@ static void BM_IsolatedFilter(benchmark::State& state) { grpc_core::ExecCtx exec_ctx; size_t channel_size = grpc_channel_stack_size( - filters.size() == 0 ? nullptr : filters.data(), filters.size()); + filters.size() == 0 ? nullptr : &filters[0], filters.size()); grpc_channel_stack* channel_stack = static_cast(gpr_zalloc(channel_size)); GPR_ASSERT(GRPC_LOG_IF_ERROR( diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 72b247c48da..0e65bf6daa0 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -1084,6 +1084,7 @@ src/core/lib/iomgr/buffer_list.h \ src/core/lib/iomgr/call_combiner.h \ src/core/lib/iomgr/closure.h \ src/core/lib/iomgr/combiner.h \ +src/core/lib/iomgr/dynamic_annotations.h \ src/core/lib/iomgr/endpoint.h \ src/core/lib/iomgr/endpoint_pair.h \ src/core/lib/iomgr/error.h \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 0f1943de25f..dd5bead58ca 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1182,6 +1182,7 @@ src/core/lib/iomgr/call_combiner.h \ src/core/lib/iomgr/closure.h \ src/core/lib/iomgr/combiner.cc \ src/core/lib/iomgr/combiner.h \ +src/core/lib/iomgr/dynamic_annotations.h \ src/core/lib/iomgr/endpoint.cc \ src/core/lib/iomgr/endpoint.h \ src/core/lib/iomgr/endpoint_pair.h \ diff --git a/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh b/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh index 74778d9d291..d35bbbd2756 100755 --- a/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh +++ b/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh @@ -15,8 +15,8 @@ set -ex -# A temporary solution to give Kokoro credentials. -# The file name 4321_grpc-testing-service needs to match auth_credential in +# A temporary solution to give Kokoro credentials. +# The file name 4321_grpc-testing-service needs to match auth_credential in # the build config. mkdir -p ${KOKORO_KEYSTORE_DIR} cp ${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json ${KOKORO_KEYSTORE_DIR}/4321_grpc-testing-service @@ -35,14 +35,15 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_rc # to get "bazel" link for kokoro build, we need to generate -# invocation UUID, set an env var for bazel to pick it up -# and upload "bazel_invocation_ids" file as artifact. -export BAZEL_INTERNAL_INVOCATION_ID="$(uuidgen)" -echo "${BAZEL_INTERNAL_INVOCATION_ID}" >"${KOKORO_ARTIFACTS_DIR}/bazel_invocation_ids" +# invocation UUID, set a flag for bazel to use it +# and upload "bazel_invocation_ids" file as artifact. +BAZEL_INVOCATION_ID="$(uuidgen)" +echo "${BAZEL_INVOCATION_ID}" >"${KOKORO_ARTIFACTS_DIR}/bazel_invocation_ids" bazel \ --bazelrc=tools/remote_build/kokoro.bazelrc \ test \ + --invocation_id="${BAZEL_INVOCATION_ID}" \ $@ \ -- //test/... || FAILED="true" diff --git a/tools/remote_build/kokoro.bazelrc b/tools/remote_build/kokoro.bazelrc index 11462bd301c..2fbdd3ce020 100644 --- a/tools/remote_build/kokoro.bazelrc +++ b/tools/remote_build/kokoro.bazelrc @@ -26,7 +26,6 @@ build --auth_credentials=/tmpfs/src/keystore/4321_grpc-testing-service build --auth_scope=https://www.googleapis.com/auth/cloud-source-tools build --bes_backend=buildeventservice.googleapis.com -build --bes_best_effort=false build --bes_timeout=600s build --project_id=grpc-testing diff --git a/tools/run_tests/artifacts/build_package_python.sh b/tools/run_tests/artifacts/build_package_python.sh index d93e8979fcc..29801a5b867 100755 --- a/tools/run_tests/artifacts/build_package_python.sh +++ b/tools/run_tests/artifacts/build_package_python.sh @@ -19,20 +19,10 @@ cd "$(dirname "$0")/../../.." mkdir -p artifacts/ +# All the python packages have been built in the artifact phase already +# and we only collect them here to deliver them to the distribtest phase. cp -r "${EXTERNAL_GIT_ROOT}"/input_artifacts/python_*/* artifacts/ || true -strip_binary_wheel() { - TEMP_WHEEL_DIR=$(mktemp -d) - unzip "$1" -d "$TEMP_WHEEL_DIR" - find "$TEMP_WHEEL_DIR" -name "_protoc_compiler*.so" -exec strip --strip-debug {} ";" - find "$TEMP_WHEEL_DIR" -name "cygrpc*.so" -exec strip --strip-debug {} ";" - (cd "$TEMP_WHEEL_DIR" && zip -r - .) > "$1" -} - -for wheel in artifacts/*.whl; do - strip_binary_wheel "$wheel" -done - # TODO: all the artifact builder configurations generate a grpcio-VERSION.tar.gz # source distribution package, and only one of them will end up # in the artifacts/ directory. They should be all equivalent though. diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index d4d5d14f079..a7231554e3d 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -9738,6 +9738,7 @@ "src/core/lib/iomgr/call_combiner.h", "src/core/lib/iomgr/closure.h", "src/core/lib/iomgr/combiner.h", + "src/core/lib/iomgr/dynamic_annotations.h", "src/core/lib/iomgr/endpoint.h", "src/core/lib/iomgr/endpoint_pair.h", "src/core/lib/iomgr/error.h", @@ -9890,6 +9891,7 @@ "src/core/lib/iomgr/call_combiner.h", "src/core/lib/iomgr/closure.h", "src/core/lib/iomgr/combiner.h", + "src/core/lib/iomgr/dynamic_annotations.h", "src/core/lib/iomgr/endpoint.h", "src/core/lib/iomgr/endpoint_pair.h", "src/core/lib/iomgr/error.h",