From 477a1aec5a222e04555113b2e0b38ddfdc463933 Mon Sep 17 00:00:00 2001 From: Dan Rosart Date: Tue, 26 Jun 2018 23:56:43 -0700 Subject: [PATCH 01/61] Don't segfault when product name from BIOS is empty Somehow some Dell servers we're trying to run a gRPC client on have an empty product name in the BIOS. When gRPC tries to creadte default credentials, it checks whether it's running on GCE by strcmp()ing the contents of /sys/class/dmi/id/product_name to some magic strings. When it reads that file, it gets only a newline; in trim() it skips over the newline in both directions, and since end < start it returns nullptr. This causes a segfault in the strcmp() call. Since a machine without a product name clearly isn't GCE, change it to return false instead. --- .../security/credentials/alts/check_gcp_environment_linux.cc | 5 +++-- test/core/security/check_gcp_environment_linux_test.cc | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/core/lib/security/credentials/alts/check_gcp_environment_linux.cc b/src/core/lib/security/credentials/alts/check_gcp_environment_linux.cc index 7c4d7a71cd1..e6b05a28f34 100644 --- a/src/core/lib/security/credentials/alts/check_gcp_environment_linux.cc +++ b/src/core/lib/security/credentials/alts/check_gcp_environment_linux.cc @@ -41,8 +41,9 @@ namespace internal { bool check_bios_data(const char* bios_data_file) { char* bios_data = read_bios_file(bios_data_file); - bool result = (!strcmp(bios_data, GRPC_ALTS_EXPECT_NAME_GOOGLE)) || - (!strcmp(bios_data, GRPC_ALTS_EXPECT_NAME_GCE)); + bool result = bios_data && + ((!strcmp(bios_data, GRPC_ALTS_EXPECT_NAME_GOOGLE)) || + (!strcmp(bios_data, GRPC_ALTS_EXPECT_NAME_GCE))); gpr_free(bios_data); return result; } diff --git a/test/core/security/check_gcp_environment_linux_test.cc b/test/core/security/check_gcp_environment_linux_test.cc index 3acd5b6ae48..b01471abd34 100644 --- a/test/core/security/check_gcp_environment_linux_test.cc +++ b/test/core/security/check_gcp_environment_linux_test.cc @@ -69,6 +69,7 @@ static void test_gcp_environment_check_failure() { GPR_ASSERT(!check_bios_data_linux_test("Amazon")); GPR_ASSERT(!check_bios_data_linux_test("Google-Chrome\t\t")); GPR_ASSERT(!check_bios_data_linux_test("Amazon")); + GPR_ASSERT(!check_bios_data_linux_test("\n")); } int main(int argc, char** argv) { From d4c98676e359e74007b4f79e264b2f8fcfc30a58 Mon Sep 17 00:00:00 2001 From: Dan Rosart Date: Fri, 6 Jul 2018 11:25:47 -0700 Subject: [PATCH 02/61] Run clang-format --- .../credentials/alts/check_gcp_environment_linux.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/lib/security/credentials/alts/check_gcp_environment_linux.cc b/src/core/lib/security/credentials/alts/check_gcp_environment_linux.cc index e6b05a28f34..8454fd7558c 100644 --- a/src/core/lib/security/credentials/alts/check_gcp_environment_linux.cc +++ b/src/core/lib/security/credentials/alts/check_gcp_environment_linux.cc @@ -41,9 +41,9 @@ namespace internal { bool check_bios_data(const char* bios_data_file) { char* bios_data = read_bios_file(bios_data_file); - bool result = bios_data && - ((!strcmp(bios_data, GRPC_ALTS_EXPECT_NAME_GOOGLE)) || - (!strcmp(bios_data, GRPC_ALTS_EXPECT_NAME_GCE))); + bool result = + bios_data && ((!strcmp(bios_data, GRPC_ALTS_EXPECT_NAME_GOOGLE)) || + (!strcmp(bios_data, GRPC_ALTS_EXPECT_NAME_GCE))); gpr_free(bios_data); return result; } From 7a7e09cf8a875226c55966e2ae241a145bb8e3ee Mon Sep 17 00:00:00 2001 From: Alexander Houben Date: Sat, 30 Jun 2018 20:45:33 +0200 Subject: [PATCH 03/61] Xamarin support: introduce PlatformApis.isXamarin*, working NativeMethods for Xamarin.Android, do not use System.Runtime.Loader on Android for the moment --- src/csharp/Grpc.Core/Grpc.Core.csproj | 15 +++++++-- src/csharp/Grpc.Core/GrpcEnvironment.cs | 3 +- .../Grpc.Core/Internal/NativeExtension.cs | 18 +++++++++- src/csharp/Grpc.Core/Internal/PlatformApis.cs | 33 +++++++++++++++++++ src/csharp/Grpc.Core/Version.csproj.include | 2 +- .../build/MonoAndroid/Grpc.Core.targets | 21 ++++++++++++ .../{ => build/net45}/Grpc.Core.targets | 0 .../build_native_ext_for_android.sh | 23 +++++++++---- 8 files changed, 104 insertions(+), 11 deletions(-) create mode 100644 src/csharp/Grpc.Core/build/MonoAndroid/Grpc.Core.targets rename src/csharp/Grpc.Core/{ => build/net45}/Grpc.Core.targets (100%) diff --git a/src/csharp/Grpc.Core/Grpc.Core.csproj b/src/csharp/Grpc.Core/Grpc.Core.csproj index 6d44be7ddd6..315d54b7c60 100755 --- a/src/csharp/Grpc.Core/Grpc.Core.csproj +++ b/src/csharp/Grpc.Core/Grpc.Core.csproj @@ -46,8 +46,12 @@ runtimes/win/native/grpc_csharp_ext.x86.dll true - - build/net45/ + + runtimes/monoandroid/armeabi-v7a/libgrpc_csharp_ext.so + true + + + runtimes/monoandroid/arm64-v8a/libgrpc_csharp_ext.so true @@ -69,4 +73,11 @@ + + + true + build + + + diff --git a/src/csharp/Grpc.Core/GrpcEnvironment.cs b/src/csharp/Grpc.Core/GrpcEnvironment.cs index 6bb2f6c3e54..27a91c723a4 100644 --- a/src/csharp/Grpc.Core/GrpcEnvironment.cs +++ b/src/csharp/Grpc.Core/GrpcEnvironment.cs @@ -423,7 +423,8 @@ namespace Grpc.Core if (!hooksRegistered) { #if NETSTANDARD1_5 - System.Runtime.Loader.AssemblyLoadContext.Default.Unloading += (assemblyLoadContext) => { HandleShutdown(); }; + // FIXME couldn't get around a "cannot resolve type" runtime exception on Xamarin.Android + //System.Runtime.Loader.AssemblyLoadContext.Default.Unloading += (assemblyLoadContext) => { HandleShutdown(); }; #else AppDomain.CurrentDomain.ProcessExit += (sender, eventArgs) => { HandleShutdown(); }; AppDomain.CurrentDomain.DomainUnload += (sender, eventArgs) => { HandleShutdown(); }; diff --git a/src/csharp/Grpc.Core/Internal/NativeExtension.cs b/src/csharp/Grpc.Core/Internal/NativeExtension.cs index d5ec998bbdb..5e1affcb996 100644 --- a/src/csharp/Grpc.Core/Internal/NativeExtension.cs +++ b/src/csharp/Grpc.Core/Internal/NativeExtension.cs @@ -106,7 +106,9 @@ namespace Grpc.Core.Internal /// private static NativeMethods LoadNativeMethods() { - return PlatformApis.IsUnity ? LoadNativeMethodsUnity() : new NativeMethods(LoadUnmanagedLibrary()); + return PlatformApis.IsUnity ? LoadNativeMethodsUnity() : + PlatformApis.IsXamarin ? LoadNativeMethodsXamarin() : + new NativeMethods(LoadUnmanagedLibrary()); } /// @@ -128,6 +130,20 @@ namespace Grpc.Core.Internal } } + /// + /// Return native method delegates when running on the Xamarin platform. + /// WARNING: Xamarin support is experimental and work-in-progress. Don't expect it to work. + /// + private static NativeMethods LoadNativeMethodsXamarin() + { + if (PlatformApis.IsXamarinAndroid) + { + return new NativeMethods(new NativeMethods.DllImportsFromSharedLib()); + } + // not tested yet + return new NativeMethods(new NativeMethods.DllImportsFromStaticLib()); + } + private static string GetAssemblyPath() { var assembly = typeof(NativeExtension).GetTypeInfo().Assembly; diff --git a/src/csharp/Grpc.Core/Internal/PlatformApis.cs b/src/csharp/Grpc.Core/Internal/PlatformApis.cs index b90fbccb2bd..6c4ee0bdb79 100644 --- a/src/csharp/Grpc.Core/Internal/PlatformApis.cs +++ b/src/csharp/Grpc.Core/Internal/PlatformApis.cs @@ -33,12 +33,17 @@ namespace Grpc.Core.Internal internal static class PlatformApis { const string UnityEngineApplicationClassName = "UnityEngine.Application, UnityEngine"; + const string XamarinAndroidActivityClassName = "Android.App.Activity, Mono.Android"; + const string XamariniOSEnumClassName = "Mono.CSharp.Enum, Mono.CSharp"; static readonly bool isLinux; static readonly bool isMacOSX; static readonly bool isWindows; static readonly bool isMono; static readonly bool isNetCore; static readonly bool isUnity; + static readonly bool isXamarin; + static readonly bool isXamariniOS; + static readonly bool isXamarinAndroid; static PlatformApis() { @@ -58,6 +63,9 @@ namespace Grpc.Core.Internal #endif isMono = Type.GetType("Mono.Runtime") != null; isUnity = Type.GetType(UnityEngineApplicationClassName) != null; + isXamariniOS = Type.GetType(XamariniOSEnumClassName) != null; + isXamarinAndroid = Type.GetType(XamarinAndroidActivityClassName) != null; + isXamarin = isXamariniOS || isXamarinAndroid; } public static bool IsLinux @@ -88,6 +96,31 @@ namespace Grpc.Core.Internal get { return isUnity; } } + /// + /// true if running on a Xamarin platform (either Xamarin.Android or Xamarin.iOS), + /// false otherwise. + /// + public static bool IsXamarin + { + get { return isXamarin; } + } + + /// + /// true if running on Xamarin.iOS, false otherwise. + /// + public static bool IsXamariniOS + { + get { return isXamariniOS; } + } + + /// + /// true if running on Xamarin.Android, false otherwise. + /// + public static bool IsXamarinAndroid + { + get { return isXamarinAndroid; } + } + /// /// true if running on .NET Core (CoreCLR), false otherwise. /// diff --git a/src/csharp/Grpc.Core/Version.csproj.include b/src/csharp/Grpc.Core/Version.csproj.include index 2b45e7ae203..cadf091e992 100755 --- a/src/csharp/Grpc.Core/Version.csproj.include +++ b/src/csharp/Grpc.Core/Version.csproj.include @@ -1,7 +1,7 @@ - 1.14.0-dev + 1.14.1-dev 3.5.1 diff --git a/src/csharp/Grpc.Core/build/MonoAndroid/Grpc.Core.targets b/src/csharp/Grpc.Core/build/MonoAndroid/Grpc.Core.targets new file mode 100644 index 00000000000..f764f4cae1d --- /dev/null +++ b/src/csharp/Grpc.Core/build/MonoAndroid/Grpc.Core.targets @@ -0,0 +1,21 @@ + + + + <_GrpcCoreNugetNativePath Condition="'$(_GrpcCoreNugetNativePath)' == ''">$(MSBuildThisFileDirectory)..\..\ + + + + + Always + arm64-v8a + + + + + + Always + armeabi-v7a + + + + diff --git a/src/csharp/Grpc.Core/Grpc.Core.targets b/src/csharp/Grpc.Core/build/net45/Grpc.Core.targets similarity index 100% rename from src/csharp/Grpc.Core/Grpc.Core.targets rename to src/csharp/Grpc.Core/build/net45/Grpc.Core.targets diff --git a/src/csharp/experimental/build_native_ext_for_android.sh b/src/csharp/experimental/build_native_ext_for_android.sh index 8197df7c536..4b4b26524fd 100755 --- a/src/csharp/experimental/build_native_ext_for_android.sh +++ b/src/csharp/experimental/build_native_ext_for_android.sh @@ -23,17 +23,28 @@ mkdir -p build cd build # set to the location where Android SDK is installed -# e.g. ANDROID_NDK_PATH="$HOME/android-ndk-r16b" +ANDROID_SDK_PATH="$HOME/Android/Sdk" -cmake ../.. \ - -DCMAKE_SYSTEM_NAME=Android \ - -DCMAKE_SYSTEM_VERSION=15 \ - -DCMAKE_ANDROID_ARCH_ABI=armeabi-v7a \ +# set to location where Android NDK is installed, usually a subfolder of Android SDK +# install the Android NDK through the Android SDK Manager +ANDROID_NDK_PATH=${ANDROID_SDK_PATH}/ndk-bundle + +# set to location of the cmake executable +# by default, use cmake binary from the Android SDK +CMAKE_PATH="${ANDROID_SDK_PATH}/cmake/3.6.4111459/bin/cmake" + +# ANDROID_ABI in ('arm64-v8a', 'armeabi-v7a') +${CMAKE_PATH} ../.. \ + -DCMAKE_TOOLCHAIN_FILE="${ANDROID_NDK_PATH}/build/cmake/android.toolchain.cmake" \ -DCMAKE_ANDROID_NDK="${ANDROID_NDK_PATH}" \ -DCMAKE_ANDROID_STL_TYPE=c++_static \ -DRUN_HAVE_POSIX_REGEX=0 \ -DRUN_HAVE_STD_REGEX=0 \ -DRUN_HAVE_STEADY_CLOCK=0 \ - -DCMAKE_BUILD_TYPE=Release + -DCMAKE_BUILD_TYPE=Release \ + -DANDROID_PLATFORM=android-28 \ + -DANDROID_ABI=arm64-v8a \ + -DANDROID_NDK="${ANDROID_NDK_PATH}" make -j4 grpc_csharp_ext + From 5d24ab9faa32af6fcf6295d4612625b9267cccb1 Mon Sep 17 00:00:00 2001 From: jiangtaoli2016 Date: Thu, 12 Jul 2018 14:02:50 -0700 Subject: [PATCH 04/61] ssl_check_peer bypass ALPN check if NPN is used --- .../security/security_connector/security_connector.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/core/lib/security/security_connector/security_connector.cc b/src/core/lib/security/security_connector/security_connector.cc index cc72bb61649..59cf3a0af18 100644 --- a/src/core/lib/security/security_connector/security_connector.cc +++ b/src/core/lib/security/security_connector/security_connector.cc @@ -57,6 +57,10 @@ static const char* installed_roots_path = INSTALL_PREFIX "/share/grpc/roots.pem"; #endif +#ifndef TSI_OPENSSL_ALPN_SUPPORT +#define TSI_OPENSSL_ALPN_SUPPORT 1 +#endif + /* -- Overridden default roots. -- */ static grpc_ssl_roots_override_callback ssl_roots_override_cb = nullptr; @@ -850,7 +854,8 @@ grpc_auth_context* grpc_ssl_peer_to_auth_context(const tsi_peer* peer) { static grpc_error* ssl_check_peer(grpc_security_connector* sc, const char* peer_name, const tsi_peer* peer, grpc_auth_context** auth_context) { - /* Check the ALPN. */ +#if TSI_OPENSSL_ALPN_SUPPORT + /* Check the ALPN if ALPN is supported. */ const tsi_peer_property* p = tsi_peer_get_property_by_name(peer, TSI_SSL_ALPN_SELECTED_PROTOCOL); if (p == nullptr) { @@ -861,7 +866,7 @@ static grpc_error* ssl_check_peer(grpc_security_connector* sc, return GRPC_ERROR_CREATE_FROM_STATIC_STRING( "Cannot check peer: invalid ALPN value."); } - +#endif /* TSI_OPENSSL_ALPN_SUPPORT */ /* Check the peer name if specified. */ if (peer_name != nullptr && !grpc_ssl_host_matches_name(peer, peer_name)) { char* msg; From 37d8bbc32dd7929682f9dacd4b7041f76f169877 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Tue, 10 Jul 2018 13:30:57 -0700 Subject: [PATCH 05/61] resolver and default executors --- src/core/lib/iomgr/executor.cc | 114 ++++++++++++------ src/core/lib/iomgr/executor.h | 40 +++++- src/core/lib/iomgr/resolve_address_posix.cc | 5 +- src/core/lib/iomgr/resolve_address_windows.cc | 5 +- 4 files changed, 121 insertions(+), 43 deletions(-) diff --git a/src/core/lib/iomgr/executor.cc b/src/core/lib/iomgr/executor.cc index 1ad13b831dd..d87eb4fbf85 100644 --- a/src/core/lib/iomgr/executor.cc +++ b/src/core/lib/iomgr/executor.cc @@ -44,7 +44,7 @@ grpc_core::TraceFlag executor_trace(false, "executor"); GPR_TLS_DECL(g_this_thread_state); -GrpcExecutor::GrpcExecutor(const char* executor_name) : name_(executor_name) { +GrpcExecutor::GrpcExecutor(const char* name) : name_(name) { adding_thread_lock_ = GPR_SPINLOCK_STATIC_INITIALIZER; gpr_atm_no_barrier_store(&num_threads_, 0); max_threads_ = GPR_MAX(1, 2 * gpr_cpu_num_cores()); @@ -298,60 +298,104 @@ void GrpcExecutor::Enqueue(grpc_closure* closure, grpc_error* error, } while (retry_push); } -static GrpcExecutor* global_executor; +static GrpcExecutor* executors[GRPC_NUM_EXECUTORS]; -void enqueue_long(grpc_closure* closure, grpc_error* error) { - global_executor->Enqueue(closure, error, false /* is_short */); +void default_enqueue_short(grpc_closure* closure, grpc_error* error) { + executors[GRPC_DEFAULT_EXECUTOR]->Enqueue(closure, error, + true /* is_short */); } -void enqueue_short(grpc_closure* closure, grpc_error* error) { - global_executor->Enqueue(closure, error, true /* is_short */); +void default_enqueue_long(grpc_closure* closure, grpc_error* error) { + executors[GRPC_DEFAULT_EXECUTOR]->Enqueue(closure, error, + false /* is_short */); } -// Short-Job executor scheduler -static const grpc_closure_scheduler_vtable global_executor_vtable_short = { - enqueue_short, enqueue_short, "executor-short"}; -static grpc_closure_scheduler global_scheduler_short = { - &global_executor_vtable_short}; +void resolver_enqueue_short(grpc_closure* closure, grpc_error* error) { + executors[GRPC_RESOLVER_EXECUTOR]->Enqueue(closure, error, + true /* is_short */); +} + +void resolver_enqueue_long(grpc_closure* closure, grpc_error* error) { + executors[GRPC_RESOLVER_EXECUTOR]->Enqueue(closure, error, + false /* is_short */); +} -// Long-job executor scheduler -static const grpc_closure_scheduler_vtable global_executor_vtable_long = { - enqueue_long, enqueue_long, "executor-long"}; -static grpc_closure_scheduler global_scheduler_long = { - &global_executor_vtable_long}; +static const grpc_closure_scheduler_vtable vtables_[] = { + {&default_enqueue_short, &default_enqueue_short, "def-ex-short"}, + {&default_enqueue_long, &default_enqueue_long, "def-ex-long"}, + {&resolver_enqueue_short, &resolver_enqueue_short, "res-ex-short"}, + {&resolver_enqueue_long, &resolver_enqueue_long, "res-ex-long"}}; + +static grpc_closure_scheduler schedulers_[] = { + {&vtables_[0]}, // Default short + {&vtables_[1]}, // Default long + {&vtables_[2]}, // Resolver short + {&vtables_[3]} // Resolver long +}; + +const char* executor_name(GrpcExecutorType executor_type) { + switch (executor_type) { + case GRPC_DEFAULT_EXECUTOR: + return "default-executor"; + case GRPC_RESOLVER_EXECUTOR: + return "resolver-executor"; + default: + GPR_UNREACHABLE_CODE(return "unknown"); + } + GPR_UNREACHABLE_CODE(return "unknown"); +} // grpc_executor_init() and grpc_executor_shutdown() functions are called in the // the grpc_init() and grpc_shutdown() code paths which are protected by a // global mutex. So it is okay to assume that these functions are thread-safe void grpc_executor_init() { - if (global_executor != nullptr) { - // grpc_executor_init() already called once (and grpc_executor_shutdown() - // wasn't called) - return; + for (int i = 0; i < GRPC_NUM_EXECUTORS; i++) { + // Return if grpc_executor_init() already called earlier + if (executors[i] != nullptr) { + GPR_ASSERT(i == 0); + break; + } + + executors[i] = grpc_core::New( + executor_name(static_cast(i))); + executors[i]->Init(); } +} - global_executor = grpc_core::New("global-executor"); - global_executor->Init(); +grpc_closure_scheduler* grpc_executor_scheduler(GrpcExecutorType executor_type, + GrpcExecutorJobType job_type) { + return &schedulers_[(executor_type * GRPC_NUM_EXECUTORS) + job_type]; +} + +grpc_closure_scheduler* grpc_executor_scheduler(GrpcExecutorJobType job_type) { + return grpc_executor_scheduler(GRPC_DEFAULT_EXECUTOR, job_type); } void grpc_executor_shutdown() { - // Shutdown already called - if (global_executor == nullptr) { - return; - } + for (int i = 0; i < GRPC_NUM_EXECUTORS; i++) { + // Return if grpc_executor_shutdown() is already called earlier + if (executors[i] == nullptr) { + GPR_ASSERT(i == 0); + break; + } - global_executor->Shutdown(); - grpc_core::Delete(global_executor); - global_executor = nullptr; + executors[i]->Shutdown(); + grpc_core::Delete(executors[i]); + executors[i] = nullptr; + } } -bool grpc_executor_is_threaded() { return global_executor->IsThreaded(); } +bool grpc_executor_is_threaded(GrpcExecutorType executor_type) { + GPR_ASSERT(executor_type < GRPC_NUM_EXECUTORS); + return executors[executor_type]->IsThreaded(); +} -void grpc_executor_set_threading(bool enable) { - global_executor->SetThreading(enable); +bool grpc_executor_is_threaded() { + return grpc_executor_is_threaded(GRPC_DEFAULT_EXECUTOR); } -grpc_closure_scheduler* grpc_executor_scheduler(GrpcExecutorJobType job_type) { - return job_type == GRPC_EXECUTOR_SHORT ? &global_scheduler_short - : &global_scheduler_long; +void grpc_executor_set_threading(bool enable) { + for (int i = 0; i < GRPC_NUM_EXECUTORS; i++) { + executors[i]->SetThreading(enable); + } } diff --git a/src/core/lib/iomgr/executor.h b/src/core/lib/iomgr/executor.h index 395fc528637..bb2c2d82b99 100644 --- a/src/core/lib/iomgr/executor.h +++ b/src/core/lib/iomgr/executor.h @@ -36,7 +36,11 @@ typedef struct { grpc_core::Thread thd; } ThreadState; -typedef enum { GRPC_EXECUTOR_SHORT, GRPC_EXECUTOR_LONG } GrpcExecutorJobType; +typedef enum { + GRPC_EXECUTOR_SHORT = 0, + GRPC_EXECUTOR_LONG, + GRPC_NUM_EXECUTOR_JOB_TYPES // Add new values above this +} GrpcExecutorJobType; class GrpcExecutor { public: @@ -70,14 +74,42 @@ class GrpcExecutor { // == Global executor functions == +typedef enum { + GRPC_DEFAULT_EXECUTOR = 0, + GRPC_RESOLVER_EXECUTOR, + + GRPC_NUM_EXECUTORS // Add new values above this +} GrpcExecutorType; + +// TODO(sreek): Currently we have two executors (available globally): The +// default executor and the resolver executor. +// +// Some of the functions below operate on the DEFAULT executor only while some +// operate of ALL the executors. This is a bit confusing and should be cleaned +// up in future (where we make all the following functions take executor_type +// and/or job_type) + +// Initialize ALL the executors void grpc_executor_init(); +// Shutdown ALL the executors +void grpc_executor_shutdown(); + +// Set the threading mode for ALL the executors +void grpc_executor_set_threading(bool enable); + +// Get the DEFAULT executor scheduler for the given job_type grpc_closure_scheduler* grpc_executor_scheduler(GrpcExecutorJobType job_type); -void grpc_executor_shutdown(); +// Get the executor scheduler for a given executor_type and a job_type +grpc_closure_scheduler* grpc_executor_scheduler(GrpcExecutorType executor_type, + GrpcExecutorJobType job_type); -bool grpc_executor_is_threaded(); +// Return if a given executor is running in threaded mode (i.e if +// grpc_executor_set_threading(true) was called previously on that executor) +bool grpc_executor_is_threaded(GrpcExecutorType executor_type); -void grpc_executor_set_threading(bool enable); +// Return if the DEFAULT executor is threaded +bool grpc_executor_is_threaded(); #endif /* GRPC_CORE_LIB_IOMGR_EXECUTOR_H */ diff --git a/src/core/lib/iomgr/resolve_address_posix.cc b/src/core/lib/iomgr/resolve_address_posix.cc index 7a825643e12..c285d7eca66 100644 --- a/src/core/lib/iomgr/resolve_address_posix.cc +++ b/src/core/lib/iomgr/resolve_address_posix.cc @@ -166,8 +166,9 @@ static void posix_resolve_address(const char* name, const char* default_port, grpc_closure* on_done, grpc_resolved_addresses** addrs) { request* r = static_cast(gpr_malloc(sizeof(request))); - GRPC_CLOSURE_INIT(&r->request_closure, do_request_thread, r, - grpc_executor_scheduler(GRPC_EXECUTOR_SHORT)); + GRPC_CLOSURE_INIT( + &r->request_closure, do_request_thread, r, + grpc_executor_scheduler(GRPC_RESOLVER_EXECUTOR, GRPC_EXECUTOR_SHORT)); r->name = gpr_strdup(name); r->default_port = gpr_strdup(default_port); r->on_done = on_done; diff --git a/src/core/lib/iomgr/resolve_address_windows.cc b/src/core/lib/iomgr/resolve_address_windows.cc index 71c92615ad6..3e977dca2da 100644 --- a/src/core/lib/iomgr/resolve_address_windows.cc +++ b/src/core/lib/iomgr/resolve_address_windows.cc @@ -151,8 +151,9 @@ static void windows_resolve_address(const char* name, const char* default_port, grpc_closure* on_done, grpc_resolved_addresses** addresses) { request* r = (request*)gpr_malloc(sizeof(request)); - GRPC_CLOSURE_INIT(&r->request_closure, do_request_thread, r, - grpc_executor_scheduler(GRPC_EXECUTOR_SHORT)); + GRPC_CLOSURE_INIT( + &r->request_closure, do_request_thread, r, + grpc_executor_scheduler(GRPC_RESOLVER_EXECUTOR, GRPC_EXECUTOR_SHORT)); r->name = gpr_strdup(name); r->default_port = gpr_strdup(default_port); r->on_done = on_done; From 00476fd2b8876f0e33f5bec4f14cd73edbc9fe8b Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Mon, 16 Jul 2018 18:09:27 -0700 Subject: [PATCH 06/61] Fix tsan issue --- src/core/lib/iomgr/executor.cc | 122 ++++++++++++++++++++++----------- src/core/lib/iomgr/executor.h | 5 +- 2 files changed, 84 insertions(+), 43 deletions(-) diff --git a/src/core/lib/iomgr/executor.cc b/src/core/lib/iomgr/executor.cc index d87eb4fbf85..3c3a784966a 100644 --- a/src/core/lib/iomgr/executor.cc +++ b/src/core/lib/iomgr/executor.cc @@ -40,19 +40,25 @@ gpr_log(GPR_INFO, "EXECUTOR " format, __VA_ARGS__); \ } +#define EXECUTOR_TRACE0(str) \ + if (executor_trace.enabled()) { \ + gpr_log(GPR_INFO, "EXECUTOR " str); \ + } + grpc_core::TraceFlag executor_trace(false, "executor"); GPR_TLS_DECL(g_this_thread_state); GrpcExecutor::GrpcExecutor(const char* name) : name_(name) { adding_thread_lock_ = GPR_SPINLOCK_STATIC_INITIALIZER; - gpr_atm_no_barrier_store(&num_threads_, 0); + gpr_atm_rel_store(&num_threads_, 0); max_threads_ = GPR_MAX(1, 2 * gpr_cpu_num_cores()); } void GrpcExecutor::Init() { SetThreading(true); } -size_t GrpcExecutor::RunClosures(grpc_closure_list list) { +size_t GrpcExecutor::RunClosures(const char* executor_name, + grpc_closure_list list) { size_t n = 0; grpc_closure* c = list.head; @@ -60,11 +66,11 @@ size_t GrpcExecutor::RunClosures(grpc_closure_list list) { grpc_closure* next = c->next_data.next; grpc_error* error = c->error_data.error; #ifndef NDEBUG - EXECUTOR_TRACE("run %p [created by %s:%d]", c, c->file_created, - c->line_created); + EXECUTOR_TRACE("(%s) run %p [created by %s:%d]", executor_name, c, + c->file_created, c->line_created); c->scheduled = false; #else - EXECUTOR_TRACE("run %p", c); + EXECUTOR_TRACE("(%s) run %p", executor_name, c); #endif c->cb(c->cb_arg, error); GRPC_ERROR_UNREF(error); @@ -77,17 +83,21 @@ size_t GrpcExecutor::RunClosures(grpc_closure_list list) { } bool GrpcExecutor::IsThreaded() const { - return gpr_atm_no_barrier_load(&num_threads_) > 0; + return gpr_atm_acq_load(&num_threads_) > 0; } void GrpcExecutor::SetThreading(bool threading) { - gpr_atm curr_num_threads = gpr_atm_no_barrier_load(&num_threads_); + gpr_atm curr_num_threads = gpr_atm_acq_load(&num_threads_); + EXECUTOR_TRACE("(%s) SetThreading(%d) begin", name_, threading); if (threading) { - if (curr_num_threads > 0) return; + if (curr_num_threads > 0) { + EXECUTOR_TRACE("(%s) SetThreading(true). curr_num_threads == 0", name_); + return; + } GPR_ASSERT(num_threads_ == 0); - gpr_atm_no_barrier_store(&num_threads_, 1); + gpr_atm_rel_store(&num_threads_, 1); gpr_tls_init(&g_this_thread_state); thd_state_ = static_cast( gpr_zalloc(sizeof(ThreadState) * max_threads_)); @@ -96,6 +106,7 @@ void GrpcExecutor::SetThreading(bool threading) { gpr_mu_init(&thd_state_[i].mu); gpr_cv_init(&thd_state_[i].cv); thd_state_[i].id = i; + thd_state_[i].name = name_; thd_state_[i].thd = grpc_core::Thread(); thd_state_[i].elems = GRPC_CLOSURE_LIST_INIT; } @@ -104,7 +115,10 @@ void GrpcExecutor::SetThreading(bool threading) { grpc_core::Thread(name_, &GrpcExecutor::ThreadMain, &thd_state_[0]); thd_state_[0].thd.Start(); } else { // !threading - if (curr_num_threads == 0) return; + if (curr_num_threads == 0) { + EXECUTOR_TRACE("(%s) SetThreading(false). curr_num_threads == 0", name_); + return; + } for (size_t i = 0; i < max_threads_; i++) { gpr_mu_lock(&thd_state_[i].mu); @@ -121,20 +135,22 @@ void GrpcExecutor::SetThreading(bool threading) { curr_num_threads = gpr_atm_no_barrier_load(&num_threads_); for (gpr_atm i = 0; i < curr_num_threads; i++) { thd_state_[i].thd.Join(); - EXECUTOR_TRACE(" Thread %" PRIdPTR " of %" PRIdPTR " joined", i, - curr_num_threads); + EXECUTOR_TRACE("(%s) Thread %" PRIdPTR " of %" PRIdPTR " joined", name_, + i + 1, curr_num_threads); } - gpr_atm_no_barrier_store(&num_threads_, 0); + gpr_atm_rel_store(&num_threads_, 0); for (size_t i = 0; i < max_threads_; i++) { gpr_mu_destroy(&thd_state_[i].mu); gpr_cv_destroy(&thd_state_[i].cv); - RunClosures(thd_state_[i].elems); + RunClosures(thd_state_[i].name, thd_state_[i].elems); } gpr_free(thd_state_); gpr_tls_destroy(&g_this_thread_state); } + + EXECUTOR_TRACE("(%s) SetThreading(%d) done", name_, threading); } void GrpcExecutor::Shutdown() { SetThreading(false); } @@ -147,8 +163,8 @@ void GrpcExecutor::ThreadMain(void* arg) { size_t subtract_depth = 0; for (;;) { - EXECUTOR_TRACE("[%" PRIdPTR "]: step (sub_depth=%" PRIdPTR ")", ts->id, - subtract_depth); + EXECUTOR_TRACE("(%s) [%" PRIdPTR "]: step (sub_depth=%" PRIdPTR ")", + ts->name, ts->id, subtract_depth); gpr_mu_lock(&ts->mu); ts->depth -= subtract_depth; @@ -159,7 +175,7 @@ void GrpcExecutor::ThreadMain(void* arg) { } if (ts->shutdown) { - EXECUTOR_TRACE("[%" PRIdPTR "]: shutdown", ts->id); + EXECUTOR_TRACE("(%s) [%" PRIdPTR "]: shutdown", ts->name, ts->id); gpr_mu_unlock(&ts->mu); break; } @@ -169,10 +185,10 @@ void GrpcExecutor::ThreadMain(void* arg) { ts->elems = GRPC_CLOSURE_LIST_INIT; gpr_mu_unlock(&ts->mu); - EXECUTOR_TRACE("[%" PRIdPTR "]: execute", ts->id); + EXECUTOR_TRACE("(%s) [%" PRIdPTR "]: execute", ts->name, ts->id); grpc_core::ExecCtx::Get()->InvalidateNow(); - subtract_depth = RunClosures(closures); + subtract_depth = RunClosures(ts->name, closures); } } @@ -188,16 +204,16 @@ void GrpcExecutor::Enqueue(grpc_closure* closure, grpc_error* error, do { retry_push = false; size_t cur_thread_count = - static_cast(gpr_atm_no_barrier_load(&num_threads_)); + static_cast(gpr_atm_acq_load(&num_threads_)); // If the number of threads is zero(i.e either the executor is not threaded // or already shutdown), then queue the closure on the exec context itself if (cur_thread_count == 0) { #ifndef NDEBUG - EXECUTOR_TRACE("schedule %p (created %s:%d) inline", closure, + EXECUTOR_TRACE("(%s) schedule %p (created %s:%d) inline", name_, closure, closure->file_created, closure->line_created); #else - EXECUTOR_TRACE("schedule %p inline", closure); + EXECUTOR_TRACE("(%s) schedule %p inline", name_, closure); #endif grpc_closure_list_append(grpc_core::ExecCtx::Get()->closure_list(), closure, error); @@ -213,18 +229,18 @@ void GrpcExecutor::Enqueue(grpc_closure* closure, grpc_error* error, } ThreadState* orig_ts = ts; - bool try_new_thread = false; + for (;;) { #ifndef NDEBUG EXECUTOR_TRACE( - "try to schedule %p (%s) (created %s:%d) to thread " + "(%s) try to schedule %p (%s) (created %s:%d) to thread " "%" PRIdPTR, - closure, is_short ? "short" : "long", closure->file_created, + name_, closure, is_short ? "short" : "long", closure->file_created, closure->line_created, ts->id); #else - EXECUTOR_TRACE("try to schedule %p (%s) to thread %" PRIdPTR, closure, - is_short ? "short" : "long", ts->id); + EXECUTOR_TRACE("(%s) try to schedule %p (%s) to thread %" PRIdPTR, name_, + closure, is_short ? "short" : "long", ts->id); #endif gpr_mu_lock(&ts->mu); @@ -236,18 +252,22 @@ void GrpcExecutor::Enqueue(grpc_closure* closure, grpc_error* error, size_t idx = ts->id; ts = &thd_state_[(idx + 1) % cur_thread_count]; if (ts == orig_ts) { - // We cycled through all the threads. Retry enqueue again (by creating - // a new thread) + // We cycled through all the threads. Retry enqueue again by creating + // a new thread + // + // TODO (sreek): There is a potential issue here. We are + // unconditionally setting try_new_thread to true here. What if the + // executor is shutdown OR if cur_thread_count is already equal to + // max_threads ? + // (Fortunately, this is not an issue yet (as of july 2018) because + // there is only one instance of long job in gRPC and hence we will + // not hit this code path) retry_push = true; - // TODO (sreek): What if the executor is shutdown OR if - // cur_thread_count is already equal to max_threads ? (currently - as - // of July 2018, we do not run in to this issue because there is only - // one instance of long job in gRPC. This has to be fixed soon) try_new_thread = true; break; } - continue; + continue; // Try the next thread-state } // == Found the thread state (i.e thread) to enqueue this closure! == @@ -277,13 +297,11 @@ void GrpcExecutor::Enqueue(grpc_closure* closure, grpc_error* error, } if (try_new_thread && gpr_spinlock_trylock(&adding_thread_lock_)) { - cur_thread_count = - static_cast(gpr_atm_no_barrier_load(&num_threads_)); + cur_thread_count = static_cast(gpr_atm_acq_load(&num_threads_)); if (cur_thread_count < max_threads_) { - // Increment num_threads (Safe to do a no_barrier_store instead of a - // cas because we always increment num_threads under the - // 'adding_thread_lock') - gpr_atm_no_barrier_store(&num_threads_, cur_thread_count + 1); + // Increment num_threads (safe to do a store instead of a cas because we + // always increment num_threads under the 'adding_thread_lock') + gpr_atm_rel_store(&num_threads_, cur_thread_count + 1); thd_state_[cur_thread_count].thd = grpc_core::Thread( name_, &GrpcExecutor::ThreadMain, &thd_state_[cur_thread_count]); @@ -349,9 +367,12 @@ const char* executor_name(GrpcExecutorType executor_type) { // the grpc_init() and grpc_shutdown() code paths which are protected by a // global mutex. So it is okay to assume that these functions are thread-safe void grpc_executor_init() { + EXECUTOR_TRACE0("grpc_executor_init() enter"); for (int i = 0; i < GRPC_NUM_EXECUTORS; i++) { // Return if grpc_executor_init() already called earlier if (executors[i] != nullptr) { + // Ideally we should also assert that all executors i.e executor[0] to + // executor[GRPC_NUM_EXECUTORS-1] are != nullptr too. GPR_ASSERT(i == 0); break; } @@ -360,6 +381,7 @@ void grpc_executor_init() { executor_name(static_cast(i))); executors[i]->Init(); } + EXECUTOR_TRACE0("grpc_executor_init() done"); } grpc_closure_scheduler* grpc_executor_scheduler(GrpcExecutorType executor_type, @@ -372,17 +394,34 @@ grpc_closure_scheduler* grpc_executor_scheduler(GrpcExecutorJobType job_type) { } void grpc_executor_shutdown() { + EXECUTOR_TRACE0("grpc_executor_shutdown() enter"); for (int i = 0; i < GRPC_NUM_EXECUTORS; i++) { // Return if grpc_executor_shutdown() is already called earlier if (executors[i] == nullptr) { + // Ideally we should also assert that all executors i.e executor[0] to + // executor[GRPC_NUM_EXECUTORS-1] are nullptr too. GPR_ASSERT(i == 0); break; } - executors[i]->Shutdown(); + } + + // Delete the executor objects. + // + // NOTE: It is important to do this in a separate loop (i.e ONLY after all the + // executors are 'Shutdown' first) because it is possible for one executor + // (that is not shutdown yet) to call Enqueue() on a different executor which + // is already shutdown. This is legal and in such cases, the Enqueue() + // operation effectively "fails" and enqueues that closure on the calling + // thread's exec_ctx. + // + // By ensuring that all executors are shutdown first, we are also ensuring + // that no thread is active across all executors. + for (int i = 0; i < GRPC_NUM_EXECUTORS; i++) { grpc_core::Delete(executors[i]); executors[i] = nullptr; } + EXECUTOR_TRACE0("grpc_executor_shutdown() done"); } bool grpc_executor_is_threaded(GrpcExecutorType executor_type) { @@ -395,6 +434,7 @@ bool grpc_executor_is_threaded() { } void grpc_executor_set_threading(bool enable) { + EXECUTOR_TRACE("grpc_executor_set_threading(%d) called", enable); for (int i = 0; i < GRPC_NUM_EXECUTORS; i++) { executors[i]->SetThreading(enable); } diff --git a/src/core/lib/iomgr/executor.h b/src/core/lib/iomgr/executor.h index bb2c2d82b99..8829138c5fa 100644 --- a/src/core/lib/iomgr/executor.h +++ b/src/core/lib/iomgr/executor.h @@ -27,7 +27,8 @@ typedef struct { gpr_mu mu; - size_t id; // For debugging purposes + size_t id; // For debugging purposes + const char* name; // Thread state name gpr_cv cv; grpc_closure_list elems; size_t depth; // Number of closures in the closure list @@ -62,7 +63,7 @@ class GrpcExecutor { void Enqueue(grpc_closure* closure, grpc_error* error, bool is_short); private: - static size_t RunClosures(grpc_closure_list list); + static size_t RunClosures(const char* executor_name, grpc_closure_list list); static void ThreadMain(void* arg); const char* name_; From 68ad431864c59faa378df78faf280606296e3a6e Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Tue, 17 Jul 2018 20:26:29 -0700 Subject: [PATCH 07/61] Store schedulers_ and vtables_ in a 2D array --- src/core/lib/iomgr/executor.cc | 102 +++++++++++++++------------------ 1 file changed, 47 insertions(+), 55 deletions(-) diff --git a/src/core/lib/iomgr/executor.cc b/src/core/lib/iomgr/executor.cc index 3c3a784966a..45d96b80eb4 100644 --- a/src/core/lib/iomgr/executor.cc +++ b/src/core/lib/iomgr/executor.cc @@ -338,55 +338,46 @@ void resolver_enqueue_long(grpc_closure* closure, grpc_error* error) { false /* is_short */); } -static const grpc_closure_scheduler_vtable vtables_[] = { - {&default_enqueue_short, &default_enqueue_short, "def-ex-short"}, - {&default_enqueue_long, &default_enqueue_long, "def-ex-long"}, - {&resolver_enqueue_short, &resolver_enqueue_short, "res-ex-short"}, - {&resolver_enqueue_long, &resolver_enqueue_long, "res-ex-long"}}; - -static grpc_closure_scheduler schedulers_[] = { - {&vtables_[0]}, // Default short - {&vtables_[1]}, // Default long - {&vtables_[2]}, // Resolver short - {&vtables_[3]} // Resolver long -}; - -const char* executor_name(GrpcExecutorType executor_type) { - switch (executor_type) { - case GRPC_DEFAULT_EXECUTOR: - return "default-executor"; - case GRPC_RESOLVER_EXECUTOR: - return "resolver-executor"; - default: - GPR_UNREACHABLE_CODE(return "unknown"); - } - GPR_UNREACHABLE_CODE(return "unknown"); -} +static const grpc_closure_scheduler_vtable + vtables_[GRPC_NUM_EXECUTORS][GRPC_NUM_EXECUTOR_JOB_TYPES] = { + {{&default_enqueue_short, &default_enqueue_short, "def-ex-short"}, + {&default_enqueue_long, &default_enqueue_long, "def-ex-long"}}, + {{&resolver_enqueue_short, &resolver_enqueue_short, "res-ex-short"}, + {&resolver_enqueue_long, &resolver_enqueue_long, "res-ex-long"}}}; + +static grpc_closure_scheduler + schedulers_[GRPC_NUM_EXECUTORS][GRPC_NUM_EXECUTOR_JOB_TYPES] = { + {{&vtables_[GRPC_DEFAULT_EXECUTOR][GRPC_EXECUTOR_SHORT]}, + {&vtables_[GRPC_DEFAULT_EXECUTOR][GRPC_EXECUTOR_LONG]}}, + {{&vtables_[GRPC_RESOLVER_EXECUTOR][GRPC_EXECUTOR_SHORT]}, + {&vtables_[GRPC_RESOLVER_EXECUTOR][GRPC_EXECUTOR_LONG]}}}; // grpc_executor_init() and grpc_executor_shutdown() functions are called in the // the grpc_init() and grpc_shutdown() code paths which are protected by a // global mutex. So it is okay to assume that these functions are thread-safe void grpc_executor_init() { EXECUTOR_TRACE0("grpc_executor_init() enter"); - for (int i = 0; i < GRPC_NUM_EXECUTORS; i++) { - // Return if grpc_executor_init() already called earlier - if (executors[i] != nullptr) { - // Ideally we should also assert that all executors i.e executor[0] to - // executor[GRPC_NUM_EXECUTORS-1] are != nullptr too. - GPR_ASSERT(i == 0); - break; - } - executors[i] = grpc_core::New( - executor_name(static_cast(i))); - executors[i]->Init(); + // Return if grpc_executor_init() is already called earlier + if (executors[GRPC_DEFAULT_EXECUTOR] != nullptr) { + GPR_ASSERT(executors[GRPC_RESOLVER_EXECUTOR] != nullptr); + return; } + + executors[GRPC_DEFAULT_EXECUTOR] = + grpc_core::New("default-executor"); + executors[GRPC_RESOLVER_EXECUTOR] = + grpc_core::New("resolver-executor"); + + executors[GRPC_DEFAULT_EXECUTOR]->Init(); + executors[GRPC_RESOLVER_EXECUTOR]->Init(); + EXECUTOR_TRACE0("grpc_executor_init() done"); } grpc_closure_scheduler* grpc_executor_scheduler(GrpcExecutorType executor_type, GrpcExecutorJobType job_type) { - return &schedulers_[(executor_type * GRPC_NUM_EXECUTORS) + job_type]; + return &schedulers_[executor_type][job_type]; } grpc_closure_scheduler* grpc_executor_scheduler(GrpcExecutorJobType job_type) { @@ -395,32 +386,33 @@ grpc_closure_scheduler* grpc_executor_scheduler(GrpcExecutorJobType job_type) { void grpc_executor_shutdown() { EXECUTOR_TRACE0("grpc_executor_shutdown() enter"); - for (int i = 0; i < GRPC_NUM_EXECUTORS; i++) { - // Return if grpc_executor_shutdown() is already called earlier - if (executors[i] == nullptr) { - // Ideally we should also assert that all executors i.e executor[0] to - // executor[GRPC_NUM_EXECUTORS-1] are nullptr too. - GPR_ASSERT(i == 0); - break; - } - executors[i]->Shutdown(); + + // Return if grpc_executor_shutdown() is already called earlier + if (executors[GRPC_DEFAULT_EXECUTOR] == nullptr) { + GPR_ASSERT(executors[GRPC_RESOLVER_EXECUTOR] == nullptr); + return; } + executors[GRPC_DEFAULT_EXECUTOR]->Shutdown(); + executors[GRPC_RESOLVER_EXECUTOR]->Shutdown(); + // Delete the executor objects. // - // NOTE: It is important to do this in a separate loop (i.e ONLY after all the - // executors are 'Shutdown' first) because it is possible for one executor - // (that is not shutdown yet) to call Enqueue() on a different executor which - // is already shutdown. This is legal and in such cases, the Enqueue() - // operation effectively "fails" and enqueues that closure on the calling - // thread's exec_ctx. + // NOTE: It is important to call Shutdown() on all executors first before + // calling Delete() because it is possible for one executor (that is not + // shutdown yet) to call Enqueue() on a different executor which is already + // shutdown. This is legal and in such cases, the Enqueue() operation + // effectively "fails" and enqueues that closure on the calling thread's + // exec_ctx. // // By ensuring that all executors are shutdown first, we are also ensuring // that no thread is active across all executors. - for (int i = 0; i < GRPC_NUM_EXECUTORS; i++) { - grpc_core::Delete(executors[i]); - executors[i] = nullptr; - } + + grpc_core::Delete(executors[GRPC_DEFAULT_EXECUTOR]); + grpc_core::Delete(executors[GRPC_RESOLVER_EXECUTOR]); + executors[GRPC_DEFAULT_EXECUTOR] = nullptr; + executors[GRPC_RESOLVER_EXECUTOR] = nullptr; + EXECUTOR_TRACE0("grpc_executor_shutdown() done"); } From c194528844ece0d18035a29453c0dd53c0bc738b Mon Sep 17 00:00:00 2001 From: ncteisen Date: Mon, 16 Jul 2018 22:24:53 -0700 Subject: [PATCH 08/61] Adds C++ Channelz Service --- BUILD | 18 + CMakeLists.txt | 107 ++++++ Makefile | 146 +++++++- build.yaml | 42 ++- grpc.gyp | 13 + src/cpp/server/channelz/channelz_service.cc | 57 +++ src/cpp/server/channelz/channelz_service.h | 43 +++ .../channelz/channelz_service_plugin.cc | 82 ++++ .../server/channelz/channelz_service_plugin.h | 54 +++ src/proto/grpc/testing/echo_messages.proto | 1 + test/cpp/end2end/BUILD | 21 ++ test/cpp/end2end/channelz_service_test.cc | 353 ++++++++++++++++++ .../generated/sources_and_headers.json | 79 +++- tools/run_tests/generated/tests.json | 24 ++ 14 files changed, 1014 insertions(+), 26 deletions(-) create mode 100644 src/cpp/server/channelz/channelz_service.cc create mode 100644 src/cpp/server/channelz/channelz_service.h create mode 100644 src/cpp/server/channelz/channelz_service_plugin.cc create mode 100644 src/cpp/server/channelz/channelz_service_plugin.h create mode 100644 test/cpp/end2end/channelz_service_test.cc diff --git a/BUILD b/BUILD index 8523bbb660f..8bb98862e33 100644 --- a/BUILD +++ b/BUILD @@ -2053,6 +2053,24 @@ grpc_cc_library( alwayslink = 1, ) +grpc_cc_library( + name = "grpcpp_channelz", + srcs = [ + "src/cpp/server/channelz/channelz_service.cc", + "src/cpp/server/channelz/channelz_service_plugin.cc", + ], + hdrs = [ + "src/cpp/server/channelz/channelz_service.h", + "src/cpp/server/channelz/channelz_service_plugin.h", + ], + language = "c++", + deps = [ + ":grpc++", + "//src/proto/grpc/channelz:channelz_proto", + ], + alwayslink = 1, +) + grpc_cc_library( name = "grpc++_test", public_hdrs = [ diff --git a/CMakeLists.txt b/CMakeLists.txt index 7222894af62..3e75d4912a6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -551,6 +551,7 @@ add_dependencies(buildtests_cxx channel_arguments_test) add_dependencies(buildtests_cxx channel_filter_test) add_dependencies(buildtests_cxx channel_trace_test) add_dependencies(buildtests_cxx channelz_registry_test) +add_dependencies(buildtests_cxx channelz_service_test) add_dependencies(buildtests_cxx channelz_test) add_dependencies(buildtests_cxx check_gcp_environment_linux_test) add_dependencies(buildtests_cxx check_gcp_environment_windows_test) @@ -4690,6 +4691,64 @@ if (gRPC_INSTALL) ) endif() + +if (gRPC_BUILD_CODEGEN) +add_library(grpcpp_channelz + src/cpp/server/channelz/channelz_service.cc + src/cpp/server/channelz/channelz_service_plugin.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/channelz/channelz.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/channelz/channelz.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/channelz/channelz.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/channelz/channelz.grpc.pb.h +) + +if(WIN32 AND MSVC) + set_target_properties(grpcpp_channelz PROPERTIES COMPILE_PDB_NAME "grpcpp_channelz" + COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) + if (gRPC_INSTALL) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/grpcpp_channelz.pdb + DESTINATION ${gRPC_INSTALL_LIBDIR} OPTIONAL + ) + endif() +endif() + +protobuf_generate_grpc_cpp( + src/proto/grpc/channelz/channelz.proto +) + +target_include_directories(grpcpp_channelz + PUBLIC $ $ + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${_gRPC_SSL_INCLUDE_DIR} + PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR} + PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR} + PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR} + PRIVATE ${_gRPC_CARES_INCLUDE_DIR} + PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR} + PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + PRIVATE ${_gRPC_NANOPB_INCLUDE_DIR} + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(grpcpp_channelz + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc++ + grpc +) + +endif (gRPC_BUILD_CODEGEN) + + +if (gRPC_INSTALL) + install(TARGETS grpcpp_channelz EXPORT gRPCTargets + RUNTIME DESTINATION ${gRPC_INSTALL_BINDIR} + LIBRARY DESTINATION ${gRPC_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${gRPC_INSTALL_LIBDIR} + ) +endif() + if (gRPC_BUILD_TESTS) if (gRPC_BUILD_CODEGEN) @@ -10873,6 +10932,54 @@ target_link_libraries(channelz_registry_test endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) +add_executable(channelz_service_test + test/cpp/end2end/channelz_service_test.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/channelz/channelz.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/channelz/channelz.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/channelz/channelz.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/channelz/channelz.grpc.pb.h + third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc +) + +protobuf_generate_grpc_cpp( + src/proto/grpc/channelz/channelz.proto +) + +target_include_directories(channelz_service_test + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${_gRPC_SSL_INCLUDE_DIR} + PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR} + PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR} + PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR} + PRIVATE ${_gRPC_CARES_INCLUDE_DIR} + PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR} + PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + PRIVATE ${_gRPC_NANOPB_INCLUDE_DIR} + PRIVATE third_party/googletest/googletest/include + PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(channelz_service_test + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpcpp_channelz + grpc++_test_util + grpc_test_util + grpc++ + grpc + gpr_test_util + gpr + ${_gRPC_GFLAGS_LIBRARIES} +) + +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) + add_executable(channelz_test test/core/channel/channelz_test.cc ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/channelz/channelz.pb.cc diff --git a/Makefile b/Makefile index 47c9dc7ccdf..723e68fbd19 100644 --- a/Makefile +++ b/Makefile @@ -1142,6 +1142,7 @@ channel_arguments_test: $(BINDIR)/$(CONFIG)/channel_arguments_test channel_filter_test: $(BINDIR)/$(CONFIG)/channel_filter_test channel_trace_test: $(BINDIR)/$(CONFIG)/channel_trace_test channelz_registry_test: $(BINDIR)/$(CONFIG)/channelz_registry_test +channelz_service_test: $(BINDIR)/$(CONFIG)/channelz_service_test channelz_test: $(BINDIR)/$(CONFIG)/channelz_test check_gcp_environment_linux_test: $(BINDIR)/$(CONFIG)/check_gcp_environment_linux_test check_gcp_environment_windows_test: $(BINDIR)/$(CONFIG)/check_gcp_environment_windows_test @@ -1378,12 +1379,12 @@ static: static_c static_cxx static_c: pc_c pc_c_unsecure cache.mk $(LIBDIR)/$(CONFIG)/libaddress_sorting.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgrpc_cronet.a $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a -static_cxx: pc_cxx pc_cxx_unsecure cache.mk $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc++_cronet.a $(LIBDIR)/$(CONFIG)/libgrpc++_error_details.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a +static_cxx: pc_cxx pc_cxx_unsecure cache.mk $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc++_cronet.a $(LIBDIR)/$(CONFIG)/libgrpc++_error_details.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpcpp_channelz.a shared: shared_c shared_cxx shared_c: pc_c pc_c_unsecure cache.mk $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)address_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc_cronet$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -shared_cxx: pc_cxx pc_cxx_unsecure cache.mk $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc++$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc++_cronet$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc++_error_details$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc++_reflection$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc++_unsecure$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) +shared_cxx: pc_cxx pc_cxx_unsecure cache.mk $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc++$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc++_cronet$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc++_error_details$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc++_reflection$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc++_unsecure$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpcpp_channelz$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) shared_csharp: shared_c $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc_csharp_ext$(SHARED_VERSION_CSHARP).$(SHARED_EXT_CSHARP) grpc_csharp_ext: shared_csharp @@ -1640,6 +1641,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/channel_filter_test \ $(BINDIR)/$(CONFIG)/channel_trace_test \ $(BINDIR)/$(CONFIG)/channelz_registry_test \ + $(BINDIR)/$(CONFIG)/channelz_service_test \ $(BINDIR)/$(CONFIG)/channelz_test \ $(BINDIR)/$(CONFIG)/check_gcp_environment_linux_test \ $(BINDIR)/$(CONFIG)/check_gcp_environment_windows_test \ @@ -1818,6 +1820,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/channel_filter_test \ $(BINDIR)/$(CONFIG)/channel_trace_test \ $(BINDIR)/$(CONFIG)/channelz_registry_test \ + $(BINDIR)/$(CONFIG)/channelz_service_test \ $(BINDIR)/$(CONFIG)/channelz_test \ $(BINDIR)/$(CONFIG)/check_gcp_environment_linux_test \ $(BINDIR)/$(CONFIG)/check_gcp_environment_windows_test \ @@ -2261,6 +2264,8 @@ test_cxx: buildtests_cxx $(Q) $(BINDIR)/$(CONFIG)/channel_trace_test || ( echo test channel_trace_test failed ; exit 1 ) $(E) "[RUN] Testing channelz_registry_test" $(Q) $(BINDIR)/$(CONFIG)/channelz_registry_test || ( echo test channelz_registry_test failed ; exit 1 ) + $(E) "[RUN] Testing channelz_service_test" + $(Q) $(BINDIR)/$(CONFIG)/channelz_service_test || ( echo test channelz_service_test failed ; exit 1 ) $(E) "[RUN] Testing channelz_test" $(Q) $(BINDIR)/$(CONFIG)/channelz_test || ( echo test channelz_test failed ; exit 1 ) $(E) "[RUN] Testing check_gcp_environment_linux_test" @@ -2459,6 +2464,8 @@ ifeq ($(CONFIG),opt) $(Q) $(STRIP) $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(E) "[STRIP] Stripping libgrpc++_unsecure.a" $(Q) $(STRIP) $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a + $(E) "[STRIP] Stripping libgrpcpp_channelz.a" + $(Q) $(STRIP) $(LIBDIR)/$(CONFIG)/libgrpcpp_channelz.a endif strip-shared_c: shared_c @@ -2487,6 +2494,8 @@ ifeq ($(CONFIG),opt) $(Q) $(STRIP) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc++_reflection$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(E) "[STRIP] Stripping $(SHARED_PREFIX)grpc++_unsecure$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP)" $(Q) $(STRIP) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc++_unsecure$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) + $(E) "[STRIP] Stripping $(SHARED_PREFIX)grpcpp_channelz$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP)" + $(Q) $(STRIP) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpcpp_channelz$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) endif strip-shared_csharp: shared_csharp @@ -2946,6 +2955,9 @@ install-static_cxx: static_cxx strip-static_cxx install-pkg-config_cxx $(E) "[INSTALL] Installing libgrpc++_unsecure.a" $(Q) $(INSTALL) -d $(prefix)/lib $(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a $(prefix)/lib/libgrpc++_unsecure.a + $(E) "[INSTALL] Installing libgrpcpp_channelz.a" + $(Q) $(INSTALL) -d $(prefix)/lib + $(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgrpcpp_channelz.a $(prefix)/lib/libgrpcpp_channelz.a @@ -3047,6 +3059,15 @@ ifeq ($(SYSTEM),MINGW32) else ifneq ($(SYSTEM),Darwin) $(Q) ln -sf $(SHARED_PREFIX)grpc++_unsecure$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpc++_unsecure.so.6 $(Q) ln -sf $(SHARED_PREFIX)grpc++_unsecure$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpc++_unsecure.so +endif + $(E) "[INSTALL] Installing $(SHARED_PREFIX)grpcpp_channelz$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP)" + $(Q) $(INSTALL) -d $(prefix)/lib + $(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpcpp_channelz$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/$(SHARED_PREFIX)grpcpp_channelz$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) +ifeq ($(SYSTEM),MINGW32) + $(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgrpcpp_channelz$(SHARED_VERSION_CPP)-dll.a $(prefix)/lib/libgrpcpp_channelz.a +else ifneq ($(SYSTEM),Darwin) + $(Q) ln -sf $(SHARED_PREFIX)grpcpp_channelz$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpcpp_channelz.so.6 + $(Q) ln -sf $(SHARED_PREFIX)grpcpp_channelz$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpcpp_channelz.so endif ifneq ($(SYSTEM),MINGW32) ifneq ($(SYSTEM),Darwin) @@ -7021,6 +7042,78 @@ ifneq ($(NO_DEPS),true) endif +LIBGRPCPP_CHANNELZ_SRC = \ + src/cpp/server/channelz/channelz_service.cc \ + src/cpp/server/channelz/channelz_service_plugin.cc \ + $(GENDIR)/src/proto/grpc/channelz/channelz.pb.cc $(GENDIR)/src/proto/grpc/channelz/channelz.grpc.pb.cc \ + +PUBLIC_HEADERS_CXX += \ + +LIBGRPCPP_CHANNELZ_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBGRPCPP_CHANNELZ_SRC)))) + + +ifeq ($(NO_SECURE),true) + +# You can't build secure libraries if you don't have OpenSSL. + +$(LIBDIR)/$(CONFIG)/libgrpcpp_channelz.a: openssl_dep_error + +$(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpcpp_channelz$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP): openssl_dep_error + +else + +ifeq ($(NO_PROTOBUF),true) + +# You can't build a C++ library if you don't have protobuf - a bit overreached, but still okay. + +$(LIBDIR)/$(CONFIG)/libgrpcpp_channelz.a: protobuf_dep_error + +$(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpcpp_channelz$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP): protobuf_dep_error + +else + +$(LIBDIR)/$(CONFIG)/libgrpcpp_channelz.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(CARES_DEP) $(ADDRESS_SORTING_DEP) $(PROTOBUF_DEP) $(LIBGRPCPP_CHANNELZ_OBJS) + $(E) "[AR] Creating $@" + $(Q) mkdir -p `dirname $@` + $(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpcpp_channelz.a + $(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libgrpcpp_channelz.a $(LIBGRPCPP_CHANNELZ_OBJS) +ifeq ($(SYSTEM),Darwin) + $(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libgrpcpp_channelz.a +endif + + + +ifeq ($(SYSTEM),MINGW32) +$(LIBDIR)/$(CONFIG)/grpcpp_channelz$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP): $(LIBGRPCPP_CHANNELZ_OBJS) $(ZLIB_DEP) $(CARES_DEP) $(ADDRESS_SORTING_DEP) $(PROTOBUF_DEP) $(LIBDIR)/$(CONFIG)/grpc++$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(LIBDIR)/$(CONFIG)/grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(OPENSSL_DEP) + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,--output-def=$(LIBDIR)/$(CONFIG)/grpcpp_channelz$(SHARED_VERSION_CPP).def -Wl,--out-implib=$(LIBDIR)/$(CONFIG)/libgrpcpp_channelz$(SHARED_VERSION_CPP)-dll.a -o $(LIBDIR)/$(CONFIG)/grpcpp_channelz$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(LIBGRPCPP_CHANNELZ_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) -lgrpc++$(SHARED_VERSION_CPP)-dll -lgrpc$(SHARED_VERSION_CORE)-dll +else +$(LIBDIR)/$(CONFIG)/libgrpcpp_channelz$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP): $(LIBGRPCPP_CHANNELZ_OBJS) $(ZLIB_DEP) $(CARES_DEP) $(ADDRESS_SORTING_DEP) $(PROTOBUF_DEP) $(LIBDIR)/$(CONFIG)/libgrpc++.$(SHARED_EXT_CPP) $(LIBDIR)/$(CONFIG)/libgrpc.$(SHARED_EXT_CORE) $(OPENSSL_DEP) + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` +ifeq ($(SYSTEM),Darwin) + $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)grpcpp_channelz$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libgrpcpp_channelz$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(LIBGRPCPP_CHANNELZ_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) -lgrpc++ -lgrpc +else + $(Q) $(LDXX) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgrpcpp_channelz.so.1 -o $(LIBDIR)/$(CONFIG)/libgrpcpp_channelz$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(LIBGRPCPP_CHANNELZ_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) -lgrpc++ -lgrpc + $(Q) ln -sf $(SHARED_PREFIX)grpcpp_channelz$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(LIBDIR)/$(CONFIG)/libgrpcpp_channelz$(SHARED_VERSION_CPP).so.1 + $(Q) ln -sf $(SHARED_PREFIX)grpcpp_channelz$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(LIBDIR)/$(CONFIG)/libgrpcpp_channelz$(SHARED_VERSION_CPP).so +endif +endif + +endif + +endif + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(LIBGRPCPP_CHANNELZ_OBJS:.o=.dep) +endif +endif +$(OBJDIR)/$(CONFIG)/src/cpp/server/channelz/channelz_service.o: $(GENDIR)/src/proto/grpc/channelz/channelz.pb.cc $(GENDIR)/src/proto/grpc/channelz/channelz.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/src/cpp/server/channelz/channelz_service_plugin.o: $(GENDIR)/src/proto/grpc/channelz/channelz.pb.cc $(GENDIR)/src/proto/grpc/channelz/channelz.grpc.pb.cc + + LIBHTTP2_CLIENT_MAIN_SRC = \ $(GENDIR)/src/proto/grpc/testing/empty.pb.cc $(GENDIR)/src/proto/grpc/testing/empty.grpc.pb.cc \ $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc \ @@ -16562,6 +16655,53 @@ endif endif +CHANNELZ_SERVICE_TEST_SRC = \ + test/cpp/end2end/channelz_service_test.cc \ + $(GENDIR)/src/proto/grpc/channelz/channelz.pb.cc $(GENDIR)/src/proto/grpc/channelz/channelz.grpc.pb.cc \ + +CHANNELZ_SERVICE_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(CHANNELZ_SERVICE_TEST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/channelz_service_test: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.5.0+. + +$(BINDIR)/$(CONFIG)/channelz_service_test: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/channelz_service_test: $(PROTOBUF_DEP) $(CHANNELZ_SERVICE_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpcpp_channelz.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(CHANNELZ_SERVICE_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpcpp_channelz.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/channelz_service_test + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/cpp/end2end/channelz_service_test.o: $(LIBDIR)/$(CONFIG)/libgrpcpp_channelz.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +$(OBJDIR)/$(CONFIG)/src/proto/grpc/channelz/channelz.o: $(LIBDIR)/$(CONFIG)/libgrpcpp_channelz.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_channelz_service_test: $(CHANNELZ_SERVICE_TEST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(CHANNELZ_SERVICE_TEST_OBJS:.o=.dep) +endif +endif +$(OBJDIR)/$(CONFIG)/test/cpp/end2end/channelz_service_test.o: $(GENDIR)/src/proto/grpc/channelz/channelz.pb.cc $(GENDIR)/src/proto/grpc/channelz/channelz.grpc.pb.cc + + CHANNELZ_TEST_SRC = \ test/core/channel/channelz_test.cc \ $(GENDIR)/src/proto/grpc/channelz/channelz.pb.cc $(GENDIR)/src/proto/grpc/channelz/channelz.grpc.pb.cc \ @@ -24517,6 +24657,8 @@ src/cpp/common/secure_channel_arguments.cc: $(OPENSSL_DEP) src/cpp/common/secure_create_auth_context.cc: $(OPENSSL_DEP) src/cpp/ext/proto_server_reflection.cc: $(OPENSSL_DEP) src/cpp/ext/proto_server_reflection_plugin.cc: $(OPENSSL_DEP) +src/cpp/server/channelz/channelz_service.cc: $(OPENSSL_DEP) +src/cpp/server/channelz/channelz_service_plugin.cc: $(OPENSSL_DEP) src/cpp/server/secure_server_credentials.cc: $(OPENSSL_DEP) src/cpp/util/core_stats.cc: $(OPENSSL_DEP) src/cpp/util/error_details.cc: $(OPENSSL_DEP) diff --git a/build.yaml b/build.yaml index 3067ca9161d..a0c3784d843 100644 --- a/build.yaml +++ b/build.yaml @@ -1111,10 +1111,6 @@ filegroups: secure: true uses: - grpc_trace -- name: grpc++_channelz_proto - language: c++ - src: - - src/proto/grpc/channelz/channelz.proto - name: grpc++_codegen_base language: c++ public_headers: @@ -1359,6 +1355,10 @@ filegroups: deps: - grpc++ - grpc +- name: grpcpp_channelz_proto + language: c++ + src: + - src/proto/grpc/channelz/channelz.proto libs: - name: address_sorting build: all @@ -1851,6 +1851,20 @@ libs: vs_project_guid: '{B6E81D84-2ACB-41B8-8781-493A944C7817}' vs_props: - protoc +- name: grpcpp_channelz + build: all + language: c++ + headers: + - src/cpp/server/channelz/channelz_service.h + - src/cpp/server/channelz/channelz_service_plugin.h + src: + - src/cpp/server/channelz/channelz_service.cc + - src/cpp/server/channelz/channelz_service_plugin.cc + deps: + - grpc++ + - grpc + filegroups: + - grpcpp_channelz_proto - name: http2_client_main build: private language: c++ @@ -4295,7 +4309,7 @@ targets: - gpr_test_util - gpr filegroups: - - grpc++_channelz_proto + - grpcpp_channelz_proto uses: - grpc++_test - name: channelz_registry_test @@ -4314,6 +4328,22 @@ targets: uses: - grpc++_test uses_polling: false +- name: channelz_service_test + gtest: true + build: test + language: c++ + src: + - test/cpp/end2end/channelz_service_test.cc + deps: + - grpcpp_channelz + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr + filegroups: + - grpcpp_channelz_proto - name: channelz_test gtest: true build: test @@ -4328,7 +4358,7 @@ targets: - gpr_test_util - gpr filegroups: - - grpc++_channelz_proto + - grpcpp_channelz_proto uses: - grpc++_test - name: check_gcp_environment_linux_test diff --git a/grpc.gyp b/grpc.gyp index 0db6afe4682..7cbd9eded78 100644 --- a/grpc.gyp +++ b/grpc.gyp @@ -1585,6 +1585,19 @@ 'src/compiler/ruby_generator.cc', ], }, + { + 'target_name': 'grpcpp_channelz', + 'type': 'static_library', + 'dependencies': [ + 'grpc++', + 'grpc', + ], + 'sources': [ + 'src/cpp/server/channelz/channelz_service.cc', + 'src/cpp/server/channelz/channelz_service_plugin.cc', + 'src/proto/grpc/channelz/channelz.proto', + ], + }, { 'target_name': 'http2_client_main', 'type': 'static_library', diff --git a/src/cpp/server/channelz/channelz_service.cc b/src/cpp/server/channelz/channelz_service.cc new file mode 100644 index 00000000000..77c175e5b8f --- /dev/null +++ b/src/cpp/server/channelz/channelz_service.cc @@ -0,0 +1,57 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "src/cpp/server/channelz/channelz_service.h" + +#include +#include + +#include +#include + +namespace grpc { + +Status ChannelzService::GetTopChannels( + ServerContext* unused, const channelz::v1::GetTopChannelsRequest* request, + channelz::v1::GetTopChannelsResponse* response) { + char* json_str = grpc_channelz_get_top_channels(request->start_channel_id()); + google::protobuf::util::Status s = + google::protobuf::util::JsonStringToMessage(json_str, response); + gpr_free(json_str); + if (s != google::protobuf::util::Status::OK) { + return Status(INTERNAL, s.ToString()); + } + return Status::OK; +} + +Status ChannelzService::GetChannel( + ServerContext* unused, const channelz::v1::GetChannelRequest* request, + channelz::v1::GetChannelResponse* response) { + char* json_str = grpc_channelz_get_channel(request->channel_id()); + google::protobuf::util::Status s = + google::protobuf::util::JsonStringToMessage(json_str, response); + gpr_free(json_str); + if (s != google::protobuf::util::Status::OK) { + return Status(INTERNAL, s.ToString()); + } + return Status::OK; +} + +} // namespace grpc diff --git a/src/cpp/server/channelz/channelz_service.h b/src/cpp/server/channelz/channelz_service.h new file mode 100644 index 00000000000..f619ea49e04 --- /dev/null +++ b/src/cpp/server/channelz/channelz_service.h @@ -0,0 +1,43 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPC_INTERNAL_CPP_SERVER_CHANNELZ_SERVICE_H +#define GRPC_INTERNAL_CPP_SERVER_CHANNELZ_SERVICE_H + +#include + +#include +#include "src/proto/grpc/channelz/channelz.grpc.pb.h" + +namespace grpc { + +class ChannelzService final : public channelz::v1::Channelz::Service { + private: + // implementation of GetTopChannels rpc + Status GetTopChannels( + ServerContext* unused, const channelz::v1::GetTopChannelsRequest* request, + channelz::v1::GetTopChannelsResponse* response) override; + // implementation of GetChannel rpc + Status GetChannel(ServerContext* unused, + const channelz::v1::GetChannelRequest* request, + channelz::v1::GetChannelResponse* response) override; +}; + +} // namespace grpc + +#endif // GRPC_INTERNAL_CPP_SERVER_CHANNELZ_SERVICE_H diff --git a/src/cpp/server/channelz/channelz_service_plugin.cc b/src/cpp/server/channelz/channelz_service_plugin.cc new file mode 100644 index 00000000000..8ea01be5b5d --- /dev/null +++ b/src/cpp/server/channelz/channelz_service_plugin.cc @@ -0,0 +1,82 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "src/cpp/server/channelz/channelz_service_plugin.h" + +#include +#include +#include + +#include "src/cpp/server/channelz/channelz_service.h" + +namespace grpc { +namespace channelz { + +ChannelzServicePlugin::ChannelzServicePlugin() + : channelz_service_(new grpc::ChannelzService()) {} + +grpc::string ChannelzServicePlugin::name() { return "channelz_service"; } + +void ChannelzServicePlugin::InitServer(grpc::ServerInitializer* si) { + si->RegisterService(channelz_service_); +} + +void ChannelzServicePlugin::Finish(grpc::ServerInitializer* si) {} + +void ChannelzServicePlugin::ChangeArguments(const grpc::string& name, + void* value) {} + +bool ChannelzServicePlugin::has_sync_methods() const { + if (channelz_service_) { + return channelz_service_->has_synchronous_methods(); + } + return false; +} + +bool ChannelzServicePlugin::has_async_methods() const { + if (channelz_service_) { + return channelz_service_->has_async_methods(); + } + return false; +} + +static std::unique_ptr< ::grpc::ServerBuilderPlugin> +CreateChannelzServicePlugin() { + return std::unique_ptr< ::grpc::ServerBuilderPlugin>( + new ChannelzServicePlugin()); +} + +void InitChannelzServiceBuilderPlugin() { + static bool already_here = false; + if (already_here) return; + already_here = true; + ::grpc::ServerBuilder::InternalAddPluginFactory(&CreateChannelzServicePlugin); +} + +// Force InitChannelzServiceBuilderPlugin() to be called at static +// initialization time. +struct StaticChannelServicePluginInitializer { + StaticChannelServicePluginInitializer() { + InitChannelzServiceBuilderPlugin(); + } +} static_channelz_service_plugin_initializer; + +} // namespace channelz +} // namespace grpc diff --git a/src/cpp/server/channelz/channelz_service_plugin.h b/src/cpp/server/channelz/channelz_service_plugin.h new file mode 100644 index 00000000000..225722c7370 --- /dev/null +++ b/src/cpp/server/channelz/channelz_service_plugin.h @@ -0,0 +1,54 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPCPP_EXT_CHANNELZ_SERVICE_PLUGIN_H +#define GRPCPP_EXT_CHANNELZ_SERVICE_PLUGIN_H + +#include + +#include +#include +#include + +#include "src/cpp/server/channelz/channelz_service.h" + +namespace grpc { +namespace channelz { + +class ChannelzServicePlugin : public ::grpc::ServerBuilderPlugin { + public: + ChannelzServicePlugin(); + ::grpc::string name() override; + void InitServer(::grpc::ServerInitializer* si) override; + void Finish(::grpc::ServerInitializer* si) override; + void ChangeArguments(const ::grpc::string& name, void* value) override; + bool has_async_methods() const override; + bool has_sync_methods() const override; + + private: + std::shared_ptr channelz_service_; +}; + +/// Add channelz server plugin to \a ServerBuilder. This function should +/// be called at static initialization time. +void InitChannelzServerBuilderPlugin(); + +} // namespace channelz +} // namespace grpc + +#endif // GRPCPP_EXT_CHANNELZ_SERVICE_PLUGIN_H diff --git a/src/proto/grpc/testing/echo_messages.proto b/src/proto/grpc/testing/echo_messages.proto index 5396a2fd39a..2f935304ab2 100644 --- a/src/proto/grpc/testing/echo_messages.proto +++ b/src/proto/grpc/testing/echo_messages.proto @@ -46,6 +46,7 @@ message RequestParams { string binary_error_details = 13; ErrorStatus expected_error = 14; int32 server_sleep_us = 15; // Amount to sleep when invoking server + int32 backend_channel_idx = 16; // which backend to send request to } message EchoRequest { diff --git a/test/cpp/end2end/BUILD b/test/cpp/end2end/BUILD index 95bb7ed229d..75dec56a602 100644 --- a/test/cpp/end2end/BUILD +++ b/test/cpp/end2end/BUILD @@ -119,6 +119,27 @@ grpc_cc_library( ], ) +grpc_cc_test( + name = "channelz_service_test", + srcs = ["channelz_service_test.cc"], + external_deps = [ + "gtest", + ], + deps = [ + ":test_service_impl", + "//:gpr", + "//:grpc", + "//:grpc++", + "//:grpcpp_channelz", + "//src/proto/grpc/channelz:channelz_proto", + "//src/proto/grpc/testing:echo_messages_proto", + "//src/proto/grpc/testing:echo_proto", + "//test/core/util:gpr_test_util", + "//test/core/util:grpc_test_util", + "//test/cpp/util:test_util", + ], +) + grpc_cc_test( name = "server_early_return_test", srcs = ["server_early_return_test.cc"], diff --git a/test/cpp/end2end/channelz_service_test.cc b/test/cpp/end2end/channelz_service_test.cc new file mode 100644 index 00000000000..6253b69cf55 --- /dev/null +++ b/test/cpp/end2end/channelz_service_test.cc @@ -0,0 +1,353 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "src/cpp/server/channelz/channelz_service_plugin.h" +#include "src/proto/grpc/channelz/channelz.grpc.pb.h" +#include "src/proto/grpc/testing/echo.grpc.pb.h" +#include "test/core/util/port.h" +#include "test/core/util/test_config.h" +#include "test/cpp/end2end/test_service_impl.h" + +#include + +using grpc::channelz::v1::GetChannelRequest; +using grpc::channelz::v1::GetChannelResponse; +using grpc::channelz::v1::GetTopChannelsRequest; +using grpc::channelz::v1::GetTopChannelsResponse; + +namespace grpc { +namespace testing { +namespace { + +// Proxy service supports N backends. Sends RPC to backend dictated by +// request->backend_channel_idx(). +class Proxy : public ::grpc::testing::EchoTestService::Service { + public: + Proxy() {} + + void AddChannelToBackend(const std::shared_ptr& channel) { + stubs_.push_back(grpc::testing::EchoTestService::NewStub(channel)); + } + + Status Echo(ServerContext* server_context, const EchoRequest* request, + EchoResponse* response) override { + std::unique_ptr client_context = + ClientContext::FromServerContext(*server_context); + size_t idx = request->param().backend_channel_idx(); + GPR_ASSERT(idx < stubs_.size()); + return stubs_[idx]->Echo(client_context.get(), *request, response); + } + + private: + std::vector> stubs_; +}; + +} // namespace + +class ChannelzServerTest : public ::testing::Test { + public: + ChannelzServerTest() {} + + void SetUp() override { + // We set up a proxy server with channelz enabled. + proxy_port_ = grpc_pick_unused_port_or_die(); + ServerBuilder proxy_builder; + grpc::string proxy_server_address = "localhost:" + to_string(proxy_port_); + proxy_builder.AddListeningPort(proxy_server_address, + InsecureServerCredentials()); + // forces channelz and channel tracing to be enabled. + proxy_builder.AddChannelArgument(GRPC_ARG_ENABLE_CHANNELZ, 1); + proxy_builder.AddChannelArgument(GRPC_ARG_MAX_CHANNEL_TRACE_EVENTS_PER_NODE, + 10); + proxy_builder.RegisterService(&proxy_service_); + proxy_server_ = proxy_builder.BuildAndStart(); + } + + // Sets the proxy up to have an arbitrary number of backends. + void ConfigureProxy(size_t num_backends) { + backends_.resize(num_backends); + for (size_t i = 0; i < num_backends; ++i) { + // create a new backend. + backends_[i].port = grpc_pick_unused_port_or_die(); + ServerBuilder backend_builder; + grpc::string backend_server_address = + "localhost:" + to_string(backends_[i].port); + backend_builder.AddListeningPort(backend_server_address, + InsecureServerCredentials()); + backends_[i].service.reset(new TestServiceImpl); + // ensure that the backend itself has channelz disabled. + backend_builder.AddChannelArgument(GRPC_ARG_ENABLE_CHANNELZ, 0); + backend_builder.RegisterService(backends_[i].service.get()); + backends_[i].server = backend_builder.BuildAndStart(); + + // set up a channel to the backend. We ensure that this channel has + // channelz enabled since these channels (proxy outbound to backends) + // are the ones that our test will actually be validating. + ChannelArguments args; + args.SetInt(GRPC_ARG_ENABLE_CHANNELZ, 1); + args.SetInt(GRPC_ARG_MAX_CHANNEL_TRACE_EVENTS_PER_NODE, 10); + std::shared_ptr channel_to_backend = CreateCustomChannel( + backend_server_address, InsecureChannelCredentials(), args); + proxy_service_.AddChannelToBackend(channel_to_backend); + } + } + + void ResetStubs() { + string target = "dns:localhost:" + to_string(proxy_port_); + ChannelArguments args; + // disable channelz. We only want to focus on proxy to backend outbound. + args.SetInt(GRPC_ARG_ENABLE_CHANNELZ, 0); + std::shared_ptr channel = + CreateCustomChannel(target, InsecureChannelCredentials(), args); + channelz_stub_ = grpc::channelz::v1::Channelz::NewStub(channel); + echo_stub_ = grpc::testing::EchoTestService::NewStub(channel); + } + + void SendSuccessfulEcho(int channel_idx) { + EchoRequest request; + EchoResponse response; + request.set_message("Hello channelz"); + request.mutable_param()->set_backend_channel_idx(channel_idx); + ClientContext context; + Status s = echo_stub_->Echo(&context, request, &response); + EXPECT_EQ(response.message(), request.message()); + EXPECT_TRUE(s.ok()); + } + + void SendFailedEcho(int channel_idx) { + EchoRequest request; + EchoResponse response; + request.set_message("Hello channelz"); + request.mutable_param()->set_backend_channel_idx(channel_idx); + auto* error = request.mutable_param()->mutable_expected_error(); + error->set_code(13); // INTERNAL + error->set_error_message("error"); + ClientContext context; + Status s = echo_stub_->Echo(&context, request, &response); + EXPECT_FALSE(s.ok()); + } + + static string to_string(const int number) { + std::stringstream strs; + strs << number; + return strs.str(); + } + + protected: + // package of data needed for each backend server. + struct BackendData { + std::unique_ptr server; + int port; + std::unique_ptr service; + }; + + std::unique_ptr channelz_stub_; + std::unique_ptr echo_stub_; + + // proxy server to ping with channelz requests. + std::unique_ptr proxy_server_; + int proxy_port_; + Proxy proxy_service_; + + // backends. All implement the echo service. + std::vector backends_; + + // ensure channel server is linked in. + channelz::ChannelzServicePlugin plugin_; +}; + +TEST_F(ChannelzServerTest, BasicTest) { + ResetStubs(); + ConfigureProxy(1); + GetTopChannelsRequest request; + GetTopChannelsResponse response; + request.set_start_channel_id(0); + ClientContext context; + Status s = channelz_stub_->GetTopChannels(&context, request, &response); + EXPECT_TRUE(s.ok()); + EXPECT_EQ(response.channel_size(), 1); +} + +TEST_F(ChannelzServerTest, HighStartId) { + ResetStubs(); + ConfigureProxy(1); + GetTopChannelsRequest request; + GetTopChannelsResponse response; + request.set_start_channel_id(10000); + ClientContext context; + Status s = channelz_stub_->GetTopChannels(&context, request, &response); + EXPECT_TRUE(s.ok()); + EXPECT_EQ(response.channel_size(), 0); +} + +TEST_F(ChannelzServerTest, SuccessfulRequestTest) { + ResetStubs(); + ConfigureProxy(1); + SendSuccessfulEcho(0); + GetChannelRequest request; + GetChannelResponse response; + request.set_channel_id(1); + ClientContext context; + Status s = channelz_stub_->GetChannel(&context, request, &response); + EXPECT_TRUE(s.ok()); + EXPECT_EQ(response.channel().data().calls_started(), 1); + EXPECT_EQ(response.channel().data().calls_succeeded(), 1); + EXPECT_EQ(response.channel().data().calls_failed(), 0); +} + +TEST_F(ChannelzServerTest, FailedRequestTest) { + ResetStubs(); + ConfigureProxy(1); + SendFailedEcho(0); + GetChannelRequest request; + GetChannelResponse response; + request.set_channel_id(1); + ClientContext context; + Status s = channelz_stub_->GetChannel(&context, request, &response); + EXPECT_TRUE(s.ok()); + EXPECT_EQ(response.channel().data().calls_started(), 1); + EXPECT_EQ(response.channel().data().calls_succeeded(), 0); + EXPECT_EQ(response.channel().data().calls_failed(), 1); +} + +TEST_F(ChannelzServerTest, ManyRequestsTest) { + ResetStubs(); + ConfigureProxy(1); + // send some RPCs + const int kNumSuccess = 10; + const int kNumFailed = 11; + for (int i = 0; i < kNumSuccess; ++i) { + SendSuccessfulEcho(0); + } + for (int i = 0; i < kNumFailed; ++i) { + SendFailedEcho(0); + } + GetChannelRequest request; + GetChannelResponse response; + request.set_channel_id(1); + ClientContext context; + Status s = channelz_stub_->GetChannel(&context, request, &response); + EXPECT_TRUE(s.ok()); + EXPECT_EQ(response.channel().data().calls_started(), + kNumSuccess + kNumFailed); + EXPECT_EQ(response.channel().data().calls_succeeded(), kNumSuccess); + EXPECT_EQ(response.channel().data().calls_failed(), kNumFailed); +} + +TEST_F(ChannelzServerTest, ManyChannels) { + ResetStubs(); + const int kNumChannels = 4; + ConfigureProxy(kNumChannels); + GetTopChannelsRequest request; + GetTopChannelsResponse response; + request.set_start_channel_id(0); + ClientContext context; + Status s = channelz_stub_->GetTopChannels(&context, request, &response); + EXPECT_TRUE(s.ok()); + EXPECT_EQ(response.channel_size(), kNumChannels); +} + +TEST_F(ChannelzServerTest, ManyRequestsManyChannels) { + ResetStubs(); + const int kNumChannels = 4; + ConfigureProxy(kNumChannels); + const int kNumSuccess = 10; + const int kNumFailed = 11; + for (int i = 0; i < kNumSuccess; ++i) { + SendSuccessfulEcho(0); + SendSuccessfulEcho(2); + } + for (int i = 0; i < kNumFailed; ++i) { + SendFailedEcho(1); + SendFailedEcho(2); + } + + // the first channel saw only successes + { + GetChannelRequest request; + GetChannelResponse response; + request.set_channel_id(1); + ClientContext context; + Status s = channelz_stub_->GetChannel(&context, request, &response); + EXPECT_TRUE(s.ok()); + EXPECT_EQ(response.channel().data().calls_started(), kNumSuccess); + EXPECT_EQ(response.channel().data().calls_succeeded(), kNumSuccess); + EXPECT_EQ(response.channel().data().calls_failed(), 0); + } + + // the second channel saw only failures + { + GetChannelRequest request; + GetChannelResponse response; + request.set_channel_id(2); + ClientContext context; + Status s = channelz_stub_->GetChannel(&context, request, &response); + EXPECT_TRUE(s.ok()); + EXPECT_EQ(response.channel().data().calls_started(), kNumFailed); + EXPECT_EQ(response.channel().data().calls_succeeded(), 0); + EXPECT_EQ(response.channel().data().calls_failed(), kNumFailed); + } + + // the third channel saw both + { + GetChannelRequest request; + GetChannelResponse response; + request.set_channel_id(3); + ClientContext context; + Status s = channelz_stub_->GetChannel(&context, request, &response); + EXPECT_TRUE(s.ok()); + EXPECT_EQ(response.channel().data().calls_started(), + kNumSuccess + kNumFailed); + EXPECT_EQ(response.channel().data().calls_succeeded(), kNumSuccess); + EXPECT_EQ(response.channel().data().calls_failed(), kNumFailed); + } + + // the fourth channel saw nothing + { + GetChannelRequest request; + GetChannelResponse response; + request.set_channel_id(4); + ClientContext context; + Status s = channelz_stub_->GetChannel(&context, request, &response); + EXPECT_TRUE(s.ok()); + EXPECT_EQ(response.channel().data().calls_started(), 0); + EXPECT_EQ(response.channel().data().calls_succeeded(), 0); + EXPECT_EQ(response.channel().data().calls_failed(), 0); + } +} + +} // namespace testing +} // namespace grpc + +int main(int argc, char** argv) { + grpc_test_init(argc, argv); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 7953fa37723..b9488269d81 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -3124,10 +3124,10 @@ "gpr_test_util", "grpc", "grpc++", - "grpc++_channelz_proto", "grpc++_test", "grpc++_test_util", - "grpc_test_util" + "grpc_test_util", + "grpcpp_channelz_proto" ], "headers": [], "is_filegroup": false, @@ -3165,10 +3165,31 @@ "gpr_test_util", "grpc", "grpc++", - "grpc++_channelz_proto", + "grpc++_test_util", + "grpc_test_util", + "grpcpp_channelz", + "grpcpp_channelz_proto" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "channelz_service_test", + "src": [ + "test/cpp/end2end/channelz_service_test.cc" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc++", "grpc++_test", "grpc++_test_util", - "grpc_test_util" + "grpc_test_util", + "grpcpp_channelz_proto" ], "headers": [], "is_filegroup": false, @@ -7528,6 +7549,28 @@ "third_party": false, "type": "lib" }, + { + "deps": [ + "grpc", + "grpc++", + "grpcpp_channelz_proto" + ], + "headers": [ + "src/cpp/server/channelz/channelz_service.h", + "src/cpp/server/channelz/channelz_service_plugin.h" + ], + "is_filegroup": false, + "language": "c++", + "name": "grpcpp_channelz", + "src": [ + "src/cpp/server/channelz/channelz_service.cc", + "src/cpp/server/channelz/channelz_service.h", + "src/cpp/server/channelz/channelz_service_plugin.cc", + "src/cpp/server/channelz/channelz_service_plugin.h" + ], + "third_party": false, + "type": "lib" + }, { "deps": [ "grpc", @@ -10888,20 +10931,6 @@ "third_party": false, "type": "filegroup" }, - { - "deps": [], - "headers": [ - "src/proto/grpc/channelz/channelz.grpc.pb.h", - "src/proto/grpc/channelz/channelz.pb.h", - "src/proto/grpc/channelz/channelz_mock.grpc.pb.h" - ], - "is_filegroup": true, - "language": "c++", - "name": "grpc++_channelz_proto", - "src": [], - "third_party": false, - "type": "filegroup" - }, { "deps": [ "grpc_codegen" @@ -11377,5 +11406,19 @@ ], "third_party": false, "type": "filegroup" + }, + { + "deps": [], + "headers": [ + "src/proto/grpc/channelz/channelz.grpc.pb.h", + "src/proto/grpc/channelz/channelz.pb.h", + "src/proto/grpc/channelz/channelz_mock.grpc.pb.h" + ], + "is_filegroup": true, + "language": "c++", + "name": "grpcpp_channelz_proto", + "src": [], + "third_party": false, + "type": "filegroup" } ] diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 5c556a2f52f..a5439a5db13 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -3755,6 +3755,30 @@ ], "uses_polling": false }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": true, + "language": "c++", + "name": "channelz_service_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "uses_polling": true + }, { "args": [], "benchmark": false, From 27b014ef351fdc87c66637793f07b88608b924af Mon Sep 17 00:00:00 2001 From: Srini Polavarapu Date: Thu, 19 Jul 2018 09:48:58 -0700 Subject: [PATCH 09/61] Bump version to 1.15.0-dev --- BUILD | 4 ++-- build.yaml | 4 ++-- doc/g_stands_for.md | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/BUILD b/BUILD index 8523bbb660f..32c01cd281e 100644 --- a/BUILD +++ b/BUILD @@ -64,11 +64,11 @@ config_setting( ) # This should be updated along with build.yaml -g_stands_for = "gladiolus" +g_stands_for = "glider" core_version = "6.0.0-dev" -version = "1.14.0-dev" +version = "1.15.0-dev" GPR_PUBLIC_HDRS = [ "include/grpc/support/alloc.h", diff --git a/build.yaml b/build.yaml index 3067ca9161d..afcbd4dcee2 100644 --- a/build.yaml +++ b/build.yaml @@ -13,8 +13,8 @@ settings: '#09': Per-language overrides are possible with (eg) ruby_version tag here '#10': See the expand_version.py for all the quirks here core_version: 6.0.0-dev - g_stands_for: gladiolus - version: 1.14.0-dev + g_stands_for: glider + version: 1.15.0-dev filegroups: - name: alts_proto headers: diff --git a/doc/g_stands_for.md b/doc/g_stands_for.md index f86bd76f328..b3ed9bd7138 100644 --- a/doc/g_stands_for.md +++ b/doc/g_stands_for.md @@ -13,4 +13,5 @@ - 1.11 'g' stands for ['gorgeous'](https://github.com/grpc/grpc/tree/v1.11.x) - 1.12 'g' stands for ['glorious'](https://github.com/grpc/grpc/tree/v1.12.x) - 1.13 'g' stands for ['gloriosa'](https://github.com/grpc/grpc/tree/v1.13.x) -- 1.14 'g' stands for ['gladiolus'](https://github.com/grpc/grpc/tree/master) +- 1.14 'g' stands for ['gladiolus'](https://github.com/grpc/grpc/tree/v1.14.x) +- 1.15 'g' stands for ['glider'](https://github.com/grpc/grpc/tree/master) From 8c9af7464c57d8a1d9cdb6505beb4baba179ee47 Mon Sep 17 00:00:00 2001 From: Srini Polavarapu Date: Thu, 19 Jul 2018 09:52:56 -0700 Subject: [PATCH 10/61] Regenerate projects --- CMakeLists.txt | 2 +- Makefile | 4 ++-- gRPC-C++.podspec | 4 ++-- gRPC-Core.podspec | 2 +- gRPC-ProtoRPC.podspec | 2 +- gRPC-RxLibrary.podspec | 2 +- gRPC.podspec | 2 +- package.xml | 4 ++-- src/core/lib/surface/version.cc | 2 +- src/cpp/common/version_cc.cc | 2 +- src/csharp/Grpc.Core/Version.csproj.include | 2 +- src/csharp/Grpc.Core/VersionInfo.cs | 4 ++-- src/csharp/build_packages_dotnetcli.bat | 2 +- src/csharp/build_packages_dotnetcli.sh | 6 +++--- src/objective-c/!ProtoCompiler-gRPCPlugin.podspec | 2 +- src/objective-c/GRPCClient/private/version.h | 2 +- src/objective-c/tests/version.h | 2 +- src/php/composer.json | 2 +- src/php/ext/grpc/version.h | 2 +- src/python/grpcio/grpc/_grpcio_metadata.py | 2 +- src/python/grpcio/grpc_version.py | 2 +- src/python/grpcio_health_checking/grpc_version.py | 2 +- src/python/grpcio_reflection/grpc_version.py | 2 +- src/python/grpcio_testing/grpc_version.py | 2 +- src/python/grpcio_tests/grpc_version.py | 2 +- src/ruby/lib/grpc/version.rb | 2 +- src/ruby/tools/version.rb | 2 +- tools/distrib/python/grpcio_tools/grpc_version.py | 2 +- tools/doxygen/Doxyfile.c++ | 2 +- tools/doxygen/Doxyfile.c++.internal | 2 +- 30 files changed, 36 insertions(+), 36 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7222894af62..e3df19be5b4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,7 +24,7 @@ cmake_minimum_required(VERSION 2.8) set(PACKAGE_NAME "grpc") -set(PACKAGE_VERSION "1.14.0-dev") +set(PACKAGE_VERSION "1.15.0-dev") set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}") set(PACKAGE_TARNAME "${PACKAGE_NAME}-${PACKAGE_VERSION}") set(PACKAGE_BUGREPORT "https://github.com/grpc/grpc/issues/") diff --git a/Makefile b/Makefile index 47c9dc7ccdf..8d0a9d91816 100644 --- a/Makefile +++ b/Makefile @@ -437,8 +437,8 @@ Q = @ endif CORE_VERSION = 6.0.0-dev -CPP_VERSION = 1.14.0-dev -CSHARP_VERSION = 1.14.0-dev +CPP_VERSION = 1.15.0-dev +CSHARP_VERSION = 1.15.0-dev CPPFLAGS_NO_ARCH += $(addprefix -I, $(INCLUDES)) $(addprefix -D, $(DEFINES)) CPPFLAGS += $(CPPFLAGS_NO_ARCH) $(ARCH_FLAGS) diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index 57d58cc4400..1d9237bf627 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -23,7 +23,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-C++' # TODO (mxyan): use version that match gRPC version when pod is stabilized - # version = '1.14.0-dev' + # version = '1.15.0-dev' version = '0.0.3' s.version = version s.summary = 'gRPC C++ library' @@ -31,7 +31,7 @@ Pod::Spec.new do |s| s.license = 'Apache License, Version 2.0' s.authors = { 'The gRPC contributors' => 'grpc-packages@google.com' } - grpc_version = '1.14.0-dev' + grpc_version = '1.15.0-dev' s.source = { :git => 'https://github.com/grpc/grpc.git', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 997617c5307..23edaec656d 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -22,7 +22,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-Core' - version = '1.14.0-dev' + version = '1.15.0-dev' s.version = version s.summary = 'Core cross-platform gRPC library, written in C' s.homepage = 'https://grpc.io' diff --git a/gRPC-ProtoRPC.podspec b/gRPC-ProtoRPC.podspec index 6548f36a32b..79ec679e3aa 100644 --- a/gRPC-ProtoRPC.podspec +++ b/gRPC-ProtoRPC.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-ProtoRPC' - version = '1.14.0-dev' + version = '1.15.0-dev' s.version = version s.summary = 'RPC library for Protocol Buffers, based on gRPC' s.homepage = 'https://grpc.io' diff --git a/gRPC-RxLibrary.podspec b/gRPC-RxLibrary.podspec index ebd942cf785..7a461e270f9 100644 --- a/gRPC-RxLibrary.podspec +++ b/gRPC-RxLibrary.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-RxLibrary' - version = '1.14.0-dev' + version = '1.15.0-dev' s.version = version s.summary = 'Reactive Extensions library for iOS/OSX.' s.homepage = 'https://grpc.io' diff --git a/gRPC.podspec b/gRPC.podspec index d6d59c21415..bf3f56bd549 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -20,7 +20,7 @@ Pod::Spec.new do |s| s.name = 'gRPC' - version = '1.14.0-dev' + version = '1.15.0-dev' s.version = version s.summary = 'gRPC client library for iOS/OSX' s.homepage = 'https://grpc.io' diff --git a/package.xml b/package.xml index a9dc2dc79ac..7f71536b1d0 100644 --- a/package.xml +++ b/package.xml @@ -13,8 +13,8 @@ 2018-01-19 - 1.14.0dev - 1.14.0dev + 1.15.0dev + 1.15.0dev beta diff --git a/src/core/lib/surface/version.cc b/src/core/lib/surface/version.cc index ac8cec29804..e92fe2c5a15 100644 --- a/src/core/lib/surface/version.cc +++ b/src/core/lib/surface/version.cc @@ -25,4 +25,4 @@ const char* grpc_version_string(void) { return "6.0.0-dev"; } -const char* grpc_g_stands_for(void) { return "gladiolus"; } +const char* grpc_g_stands_for(void) { return "glider"; } diff --git a/src/cpp/common/version_cc.cc b/src/cpp/common/version_cc.cc index a7b093dfd2a..b8fca9a6ee3 100644 --- a/src/cpp/common/version_cc.cc +++ b/src/cpp/common/version_cc.cc @@ -22,5 +22,5 @@ #include namespace grpc { -grpc::string Version() { return "1.14.0-dev"; } +grpc::string Version() { return "1.15.0-dev"; } } // namespace grpc diff --git a/src/csharp/Grpc.Core/Version.csproj.include b/src/csharp/Grpc.Core/Version.csproj.include index 2b45e7ae203..1a746acdd4f 100755 --- a/src/csharp/Grpc.Core/Version.csproj.include +++ b/src/csharp/Grpc.Core/Version.csproj.include @@ -1,7 +1,7 @@ - 1.14.0-dev + 1.15.0-dev 3.5.1 diff --git a/src/csharp/Grpc.Core/VersionInfo.cs b/src/csharp/Grpc.Core/VersionInfo.cs index c4531e5275a..295e0f2577b 100644 --- a/src/csharp/Grpc.Core/VersionInfo.cs +++ b/src/csharp/Grpc.Core/VersionInfo.cs @@ -33,11 +33,11 @@ namespace Grpc.Core /// /// Current AssemblyFileVersion of gRPC C# assemblies /// - public const string CurrentAssemblyFileVersion = "1.14.0.0"; + public const string CurrentAssemblyFileVersion = "1.15.0.0"; /// /// Current version of gRPC C# /// - public const string CurrentVersion = "1.14.0-dev"; + public const string CurrentVersion = "1.15.0-dev"; } } diff --git a/src/csharp/build_packages_dotnetcli.bat b/src/csharp/build_packages_dotnetcli.bat index 394b859e0be..67af258354f 100755 --- a/src/csharp/build_packages_dotnetcli.bat +++ b/src/csharp/build_packages_dotnetcli.bat @@ -13,7 +13,7 @@ @rem limitations under the License. @rem Current package versions -set VERSION=1.14.0-dev +set VERSION=1.15.0-dev @rem Adjust the location of nuget.exe set NUGET=C:\nuget\nuget.exe diff --git a/src/csharp/build_packages_dotnetcli.sh b/src/csharp/build_packages_dotnetcli.sh index 273d745f170..88b4d63538d 100755 --- a/src/csharp/build_packages_dotnetcli.sh +++ b/src/csharp/build_packages_dotnetcli.sh @@ -45,8 +45,8 @@ dotnet pack --configuration Release Grpc.Auth --output ../../../artifacts dotnet pack --configuration Release Grpc.HealthCheck --output ../../../artifacts dotnet pack --configuration Release Grpc.Reflection --output ../../../artifacts -nuget pack Grpc.nuspec -Version "1.14.0-dev" -OutputDirectory ../../artifacts -nuget pack Grpc.Core.NativeDebug.nuspec -Version "1.14.0-dev" -OutputDirectory ../../artifacts -nuget pack Grpc.Tools.nuspec -Version "1.14.0-dev" -OutputDirectory ../../artifacts +nuget pack Grpc.nuspec -Version "1.15.0-dev" -OutputDirectory ../../artifacts +nuget pack Grpc.Core.NativeDebug.nuspec -Version "1.15.0-dev" -OutputDirectory ../../artifacts +nuget pack Grpc.Tools.nuspec -Version "1.15.0-dev" -OutputDirectory ../../artifacts (cd ../../artifacts && zip csharp_nugets_dotnetcli.zip *.nupkg) diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index 28dfffd8631..6ad9166b32e 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -42,7 +42,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler-gRPCPlugin' - v = '1.14.0-dev' + v = '1.15.0-dev' s.version = v s.summary = 'The gRPC ProtoC plugin generates Objective-C files from .proto services.' s.description = <<-DESC diff --git a/src/objective-c/GRPCClient/private/version.h b/src/objective-c/GRPCClient/private/version.h index 60571b1f43f..52fe0dd0502 100644 --- a/src/objective-c/GRPCClient/private/version.h +++ b/src/objective-c/GRPCClient/private/version.h @@ -22,4 +22,4 @@ // instead. This file can be regenerated from the template by running // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.14.0-dev" +#define GRPC_OBJC_VERSION_STRING @"1.15.0-dev" diff --git a/src/objective-c/tests/version.h b/src/objective-c/tests/version.h index fafcac6507d..d532eed245d 100644 --- a/src/objective-c/tests/version.h +++ b/src/objective-c/tests/version.h @@ -22,5 +22,5 @@ // instead. This file can be regenerated from the template by running // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.14.0-dev" +#define GRPC_OBJC_VERSION_STRING @"1.15.0-dev" #define GRPC_C_VERSION_STRING @"6.0.0-dev" diff --git a/src/php/composer.json b/src/php/composer.json index 94918417d9f..b6716b8b4c3 100644 --- a/src/php/composer.json +++ b/src/php/composer.json @@ -2,7 +2,7 @@ "name": "grpc/grpc-dev", "description": "gRPC library for PHP - for Developement use only", "license": "Apache-2.0", - "version": "1.14.0", + "version": "1.15.0", "require": { "php": ">=5.5.0", "google/protobuf": "^v3.3.0" diff --git a/src/php/ext/grpc/version.h b/src/php/ext/grpc/version.h index c985863cb59..99dd9d68f05 100644 --- a/src/php/ext/grpc/version.h +++ b/src/php/ext/grpc/version.h @@ -20,6 +20,6 @@ #ifndef VERSION_H #define VERSION_H -#define PHP_GRPC_VERSION "1.14.0dev" +#define PHP_GRPC_VERSION "1.15.0dev" #endif /* VERSION_H */ diff --git a/src/python/grpcio/grpc/_grpcio_metadata.py b/src/python/grpcio/grpc/_grpcio_metadata.py index b336e6aae54..c33911ebc1b 100644 --- a/src/python/grpcio/grpc/_grpcio_metadata.py +++ b/src/python/grpcio/grpc/_grpcio_metadata.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc/_grpcio_metadata.py.template`!!! -__version__ = """1.14.0.dev0""" +__version__ = """1.15.0.dev0""" diff --git a/src/python/grpcio/grpc_version.py b/src/python/grpcio/grpc_version.py index 2eeaabc8b88..9337800a332 100644 --- a/src/python/grpcio/grpc_version.py +++ b/src/python/grpcio/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_version.py.template`!!! -VERSION = '1.14.0.dev0' +VERSION = '1.15.0.dev0' diff --git a/src/python/grpcio_health_checking/grpc_version.py b/src/python/grpcio_health_checking/grpc_version.py index f36de8d0fa1..3b84f7a4c55 100644 --- a/src/python/grpcio_health_checking/grpc_version.py +++ b/src/python/grpcio_health_checking/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_health_checking/grpc_version.py.template`!!! -VERSION = '1.14.0.dev0' +VERSION = '1.15.0.dev0' diff --git a/src/python/grpcio_reflection/grpc_version.py b/src/python/grpcio_reflection/grpc_version.py index 2249b07d9d7..7b0e48ea23b 100644 --- a/src/python/grpcio_reflection/grpc_version.py +++ b/src/python/grpcio_reflection/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_reflection/grpc_version.py.template`!!! -VERSION = '1.14.0.dev0' +VERSION = '1.15.0.dev0' diff --git a/src/python/grpcio_testing/grpc_version.py b/src/python/grpcio_testing/grpc_version.py index a5b275aff65..df9953fa257 100644 --- a/src/python/grpcio_testing/grpc_version.py +++ b/src/python/grpcio_testing/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_testing/grpc_version.py.template`!!! -VERSION = '1.14.0.dev0' +VERSION = '1.15.0.dev0' diff --git a/src/python/grpcio_tests/grpc_version.py b/src/python/grpcio_tests/grpc_version.py index 816dfb55bc1..b2cf129e4fb 100644 --- a/src/python/grpcio_tests/grpc_version.py +++ b/src/python/grpcio_tests/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_tests/grpc_version.py.template`!!! -VERSION = '1.14.0.dev0' +VERSION = '1.15.0.dev0' diff --git a/src/ruby/lib/grpc/version.rb b/src/ruby/lib/grpc/version.rb index 902d59bd71a..5c21a5b1685 100644 --- a/src/ruby/lib/grpc/version.rb +++ b/src/ruby/lib/grpc/version.rb @@ -14,5 +14,5 @@ # GRPC contains the General RPC module. module GRPC - VERSION = '1.14.0.dev' + VERSION = '1.15.0.dev' end diff --git a/src/ruby/tools/version.rb b/src/ruby/tools/version.rb index dad09bb965e..1b5e2c362be 100644 --- a/src/ruby/tools/version.rb +++ b/src/ruby/tools/version.rb @@ -14,6 +14,6 @@ module GRPC module Tools - VERSION = '1.14.0.dev' + VERSION = '1.15.0.dev' end end diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index ae249e73861..ccb69a8ebcb 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION = '1.14.0.dev0' +VERSION = '1.15.0.dev0' diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 322ab5eb983..2f06bda0168 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.14.0-dev +PROJECT_NUMBER = 1.15.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index ba322a90a54..a46ebe61976 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.14.0-dev +PROJECT_NUMBER = 1.15.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a From 96201183908d775e7ba58cdd79c8256baf23f167 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Thu, 19 Jul 2018 11:24:04 -0700 Subject: [PATCH 11/61] Remove blank line --- test/cpp/end2end/channelz_service_test.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/test/cpp/end2end/channelz_service_test.cc b/test/cpp/end2end/channelz_service_test.cc index 6253b69cf55..12496245845 100644 --- a/test/cpp/end2end/channelz_service_test.cc +++ b/test/cpp/end2end/channelz_service_test.cc @@ -106,7 +106,6 @@ class ChannelzServerTest : public ::testing::Test { backend_builder.AddChannelArgument(GRPC_ARG_ENABLE_CHANNELZ, 0); backend_builder.RegisterService(backends_[i].service.get()); backends_[i].server = backend_builder.BuildAndStart(); - // set up a channel to the backend. We ensure that this channel has // channelz enabled since these channels (proxy outbound to backends) // are the ones that our test will actually be validating. From c6970ad79dd77a75af5e26825abd4bcf8828cd2d Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 19 Jul 2018 16:20:49 +0200 Subject: [PATCH 12/61] fix automatized android artifact build --- .../build_native_ext_for_android.sh | 19 ++++++------ .../grpc_artifact_android_ndk/Dockerfile | 31 +++++++++++-------- .../build_artifact_csharp_android.sh | 1 + 3 files changed, 29 insertions(+), 22 deletions(-) diff --git a/src/csharp/experimental/build_native_ext_for_android.sh b/src/csharp/experimental/build_native_ext_for_android.sh index 4b4b26524fd..5687a43a4e2 100755 --- a/src/csharp/experimental/build_native_ext_for_android.sh +++ b/src/csharp/experimental/build_native_ext_for_android.sh @@ -23,18 +23,20 @@ mkdir -p build cd build # set to the location where Android SDK is installed -ANDROID_SDK_PATH="$HOME/Android/Sdk" +# e.g. ANDROID_SDK_PATH="$HOME/Android/Sdk" # set to location where Android NDK is installed, usually a subfolder of Android SDK -# install the Android NDK through the Android SDK Manager -ANDROID_NDK_PATH=${ANDROID_SDK_PATH}/ndk-bundle +# to install the Android NKD, use the "sdkmanager" tool +# e.g. ANDROID_NDK_PATH=${ANDROID_SDK_PATH}/ndk-bundle -# set to location of the cmake executable -# by default, use cmake binary from the Android SDK -CMAKE_PATH="${ANDROID_SDK_PATH}/cmake/3.6.4111459/bin/cmake" +# set to location of the cmake executable from the Android SDK +# to install cmake, use the "sdkmanager" tool +# e.g. ANDROID_SDK_CMAKE=${ANDROID_SDK_PATH}/cmake/3.6.4111459/bin/cmake # ANDROID_ABI in ('arm64-v8a', 'armeabi-v7a') -${CMAKE_PATH} ../.. \ +# e.g. ANDROID_ABI=armeabi-v7a + +${ANDROID_SDK_CMAKE} ../.. \ -DCMAKE_TOOLCHAIN_FILE="${ANDROID_NDK_PATH}/build/cmake/android.toolchain.cmake" \ -DCMAKE_ANDROID_NDK="${ANDROID_NDK_PATH}" \ -DCMAKE_ANDROID_STL_TYPE=c++_static \ @@ -43,8 +45,7 @@ ${CMAKE_PATH} ../.. \ -DRUN_HAVE_STEADY_CLOCK=0 \ -DCMAKE_BUILD_TYPE=Release \ -DANDROID_PLATFORM=android-28 \ - -DANDROID_ABI=arm64-v8a \ + -DANDROID_ABI="${ANDROID_ABI}" \ -DANDROID_NDK="${ANDROID_NDK_PATH}" make -j4 grpc_csharp_ext - diff --git a/tools/dockerfile/grpc_artifact_android_ndk/Dockerfile b/tools/dockerfile/grpc_artifact_android_ndk/Dockerfile index 77b6acfb160..be96f131f92 100644 --- a/tools/dockerfile/grpc_artifact_android_ndk/Dockerfile +++ b/tools/dockerfile/grpc_artifact_android_ndk/Dockerfile @@ -12,9 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Docker file for building gRPC artifacts. +# Docker file for building gRPC artifacts for Android. -# Recent enough cmake (>=3.9) needed by Android SDK FROM debian:sid RUN apt-get update && apt-get install -y debian-keyring && apt-key update @@ -47,20 +46,26 @@ RUN apt-get update && apt-key update && apt-get install -y \ wget \ zip && apt-get clean -# Cmake for cross-compilation -RUN apt-get update && apt-get install -y cmake golang && apt-get clean +# golang needed to build BoringSSL with cmake +RUN apt-get update && apt-get install -y golang && apt-get clean -################## -# Android NDK +# Java required by Android SDK +RUN apt-get update && apt-get -y install openjdk-8-jdk && apt-get clean -# Download and install Android NDK -RUN wget -q https://dl.google.com/android/repository/android-ndk-r16b-linux-x86_64.zip -O android_ndk.zip \ - && unzip -q android_ndk.zip \ - && rm android_ndk.zip \ - && mv ./android-ndk-r16b /opt -ENV ANDROID_NDK_PATH /opt/android-ndk-r16b +# Install Android SDK +ENV ANDROID_SDK_VERSION 4333796 +RUN mkdir -p /opt/android-sdk && cd /opt/android-sdk && \ + wget -q https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_VERSION}.zip && \ + unzip -q sdk-tools-linux-${ANDROID_SDK_VERSION}.zip && \ + rm sdk-tools-linux-${ANDROID_SDK_VERSION}.zip +ENV ANDROID_SDK_PATH /opt/android-sdk -RUN apt-get update && apt-get install -y libpthread-stubs0-dev && apt-get clean +# Install Android NDK and cmake using sdkmanager +RUN mkdir -p ~/.android && touch ~/.android/repositories.cfg +RUN yes | ${ANDROID_SDK_PATH}/tools/bin/sdkmanager --licenses # accept all licenses +RUN ${ANDROID_SDK_PATH}/tools/bin/sdkmanager ndk-bundle 'cmake;3.6.4111459' +ENV ANDROID_NDK_PATH ${ANDROID_SDK_PATH}/ndk-bundle +ENV ANDROID_SDK_CMAKE ${ANDROID_SDK_PATH}/cmake/3.6.4111459/bin/cmake RUN mkdir /var/local/jenkins diff --git a/tools/run_tests/artifacts/build_artifact_csharp_android.sh b/tools/run_tests/artifacts/build_artifact_csharp_android.sh index ba598e76a4b..e4f229e16a5 100755 --- a/tools/run_tests/artifacts/build_artifact_csharp_android.sh +++ b/tools/run_tests/artifacts/build_artifact_csharp_android.sh @@ -17,6 +17,7 @@ set -ex cd "$(dirname "$0")/../../.." +export ANDROID_ABI=armeabi-v7a src/csharp/experimental/build_native_ext_for_android.sh mkdir -p "${ARTIFACTS_OUT}" From be6f3e393fdbf80f01b311ad310ef60e5b9a6537 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 19 Jul 2018 23:26:33 +0200 Subject: [PATCH 13/61] better NativeExtension.LoadNativeMethods readability --- src/csharp/Grpc.Core/Internal/NativeExtension.cs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/csharp/Grpc.Core/Internal/NativeExtension.cs b/src/csharp/Grpc.Core/Internal/NativeExtension.cs index 5e1affcb996..f526b913af0 100644 --- a/src/csharp/Grpc.Core/Internal/NativeExtension.cs +++ b/src/csharp/Grpc.Core/Internal/NativeExtension.cs @@ -106,9 +106,15 @@ namespace Grpc.Core.Internal /// private static NativeMethods LoadNativeMethods() { - return PlatformApis.IsUnity ? LoadNativeMethodsUnity() : - PlatformApis.IsXamarin ? LoadNativeMethodsXamarin() : - new NativeMethods(LoadUnmanagedLibrary()); + if (PlatformApis.IsUnity) + { + return LoadNativeMethodsUnity(); + } + if (PlatformApis.IsXamarin) + { + return LoadNativeMethodsXamarin(); + } + return new NativeMethods(LoadUnmanagedLibrary()); } /// From 1e30bced777ffad7c9fdd78c91bdd227df2a3a62 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 19 Jul 2018 23:29:27 +0200 Subject: [PATCH 14/61] revert changes to Version.csproj.include --- src/csharp/Grpc.Core/Version.csproj.include | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/csharp/Grpc.Core/Version.csproj.include b/src/csharp/Grpc.Core/Version.csproj.include index cadf091e992..2b45e7ae203 100755 --- a/src/csharp/Grpc.Core/Version.csproj.include +++ b/src/csharp/Grpc.Core/Version.csproj.include @@ -1,7 +1,7 @@ - 1.14.1-dev + 1.14.0-dev 3.5.1 From e56556749146e5d36db56156f4f5653638b36c03 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 19 Jul 2018 23:42:14 +0200 Subject: [PATCH 15/61] explicitly list Grpc.Core.targets files --- src/csharp/Grpc.Core/Grpc.Core.csproj | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/csharp/Grpc.Core/Grpc.Core.csproj b/src/csharp/Grpc.Core/Grpc.Core.csproj index 315d54b7c60..9e5eab10328 100755 --- a/src/csharp/Grpc.Core/Grpc.Core.csproj +++ b/src/csharp/Grpc.Core/Grpc.Core.csproj @@ -54,6 +54,14 @@ runtimes/monoandroid/arm64-v8a/libgrpc_csharp_ext.so true + + build/net45/ + true + + + build/MonoAndroid/ + true + @@ -73,11 +81,4 @@ - - - true - build - - - From 6104e4f33b4d2e735ae1c69df1a3fd5d57c931a4 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Thu, 19 Jul 2018 14:59:10 -0700 Subject: [PATCH 16/61] Move ChannelzServicePlugin to public, experimental --- CMakeLists.txt | 9 +++++++++ Makefile | 1 + build.yaml | 3 ++- .../grpcpp/ext}/channelz_service_plugin.h | 4 ++++ src/cpp/server/channelz/channelz_service_plugin.cc | 5 +++-- test/cpp/end2end/channelz_service_test.cc | 4 ++-- tools/run_tests/generated/sources_and_headers.json | 8 ++++---- 7 files changed, 25 insertions(+), 9 deletions(-) rename {src/cpp/server/channelz => include/grpcpp/ext}/channelz_service_plugin.h (91%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3e75d4912a6..75f3a2f3cdf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4738,6 +4738,15 @@ target_link_libraries(grpcpp_channelz grpc ) +foreach(_hdr + include/grpcpp/ext/channelz_service_plugin.h +) + string(REPLACE "include/" "" _path ${_hdr}) + get_filename_component(_path ${_path} PATH) + install(FILES ${_hdr} + DESTINATION "${gRPC_INSTALL_INCLUDEDIR}/${_path}" + ) +endforeach() endif (gRPC_BUILD_CODEGEN) diff --git a/Makefile b/Makefile index 723e68fbd19..309187969cc 100644 --- a/Makefile +++ b/Makefile @@ -7048,6 +7048,7 @@ LIBGRPCPP_CHANNELZ_SRC = \ $(GENDIR)/src/proto/grpc/channelz/channelz.pb.cc $(GENDIR)/src/proto/grpc/channelz/channelz.grpc.pb.cc \ PUBLIC_HEADERS_CXX += \ + include/grpcpp/ext/channelz_service_plugin.h \ LIBGRPCPP_CHANNELZ_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBGRPCPP_CHANNELZ_SRC)))) diff --git a/build.yaml b/build.yaml index a0c3784d843..2ff98f12bc6 100644 --- a/build.yaml +++ b/build.yaml @@ -1854,9 +1854,10 @@ libs: - name: grpcpp_channelz build: all language: c++ + public_headers: + - include/grpcpp/ext/channelz_service_plugin.h headers: - src/cpp/server/channelz/channelz_service.h - - src/cpp/server/channelz/channelz_service_plugin.h src: - src/cpp/server/channelz/channelz_service.cc - src/cpp/server/channelz/channelz_service_plugin.cc diff --git a/src/cpp/server/channelz/channelz_service_plugin.h b/include/grpcpp/ext/channelz_service_plugin.h similarity index 91% rename from src/cpp/server/channelz/channelz_service_plugin.h rename to include/grpcpp/ext/channelz_service_plugin.h index 225722c7370..d01372b0651 100644 --- a/src/cpp/server/channelz/channelz_service_plugin.h +++ b/include/grpcpp/ext/channelz_service_plugin.h @@ -29,7 +29,10 @@ namespace grpc { namespace channelz { +namespace experimental { +// This plugin is experimental for now. Track progress in +// https://github.com/grpc/grpc/issues/15988. class ChannelzServicePlugin : public ::grpc::ServerBuilderPlugin { public: ChannelzServicePlugin(); @@ -48,6 +51,7 @@ class ChannelzServicePlugin : public ::grpc::ServerBuilderPlugin { /// be called at static initialization time. void InitChannelzServerBuilderPlugin(); +} // namespace experimental } // namespace channelz } // namespace grpc diff --git a/src/cpp/server/channelz/channelz_service_plugin.cc b/src/cpp/server/channelz/channelz_service_plugin.cc index 8ea01be5b5d..8c2972915c7 100644 --- a/src/cpp/server/channelz/channelz_service_plugin.cc +++ b/src/cpp/server/channelz/channelz_service_plugin.cc @@ -18,8 +18,7 @@ #include -#include "src/cpp/server/channelz/channelz_service_plugin.h" - +#include #include #include #include @@ -28,6 +27,7 @@ namespace grpc { namespace channelz { +namespace experimental { ChannelzServicePlugin::ChannelzServicePlugin() : channelz_service_(new grpc::ChannelzService()) {} @@ -78,5 +78,6 @@ struct StaticChannelServicePluginInitializer { } } static_channelz_service_plugin_initializer; +} // namespace experimental } // namespace channelz } // namespace grpc diff --git a/test/cpp/end2end/channelz_service_test.cc b/test/cpp/end2end/channelz_service_test.cc index 12496245845..02d36795fc8 100644 --- a/test/cpp/end2end/channelz_service_test.cc +++ b/test/cpp/end2end/channelz_service_test.cc @@ -28,7 +28,7 @@ #include #include -#include "src/cpp/server/channelz/channelz_service_plugin.h" +#include #include "src/proto/grpc/channelz/channelz.grpc.pb.h" #include "src/proto/grpc/testing/echo.grpc.pb.h" #include "test/core/util/port.h" @@ -179,7 +179,7 @@ class ChannelzServerTest : public ::testing::Test { std::vector backends_; // ensure channel server is linked in. - channelz::ChannelzServicePlugin plugin_; + channelz::experimental::ChannelzServicePlugin plugin_; }; TEST_F(ChannelzServerTest, BasicTest) { diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index b9488269d81..bf3ddb42c92 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -7556,17 +7556,17 @@ "grpcpp_channelz_proto" ], "headers": [ - "src/cpp/server/channelz/channelz_service.h", - "src/cpp/server/channelz/channelz_service_plugin.h" + "include/grpcpp/ext/channelz_service_plugin.h", + "src/cpp/server/channelz/channelz_service.h" ], "is_filegroup": false, "language": "c++", "name": "grpcpp_channelz", "src": [ + "include/grpcpp/ext/channelz_service_plugin.h", "src/cpp/server/channelz/channelz_service.cc", "src/cpp/server/channelz/channelz_service.h", - "src/cpp/server/channelz/channelz_service_plugin.cc", - "src/cpp/server/channelz/channelz_service_plugin.h" + "src/cpp/server/channelz/channelz_service_plugin.cc" ], "third_party": false, "type": "lib" From 71826b5196b1a6a2b0f446652a372cc7b8e3ae07 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 20 Jul 2018 00:06:06 +0200 Subject: [PATCH 17/61] differentiate C# android artifacts by ABI --- tools/run_tests/artifacts/artifact_targets.py | 11 ++++++++--- .../artifacts/build_artifact_csharp_android.sh | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index edde3eae28d..ada7129bfc4 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -212,11 +212,15 @@ class RubyArtifact: class CSharpExtArtifact: """Builds C# native extension library""" - def __init__(self, platform, arch): + def __init__(self, platform, arch, arch_abi=None): self.name = 'csharp_ext_%s_%s' % (platform, arch) self.platform = platform self.arch = arch + self.arch_abi = arch_abi self.labels = ['artifact', 'csharp', platform, arch] + if arch_abi: + self.name += '_%s' % arch_abi + self.labels.append(arch_abi) def pre_build_jobspecs(self): return [] @@ -227,7 +231,7 @@ class CSharpExtArtifact: self.name, 'tools/dockerfile/grpc_artifact_android_ndk', 'tools/run_tests/artifacts/build_artifact_csharp_android.sh', - environ={}) + environ={'ANDROID_ABI': self.arch_abi}) elif self.platform == 'windows': cmake_arch_option = 'Win32' if self.arch == 'x86' else self.arch return create_jobspec( @@ -348,7 +352,8 @@ def targets(): for Cls in (CSharpExtArtifact, ProtocArtifact) for platform in ('linux', 'macos', 'windows') for arch in ('x86', 'x64') ] + [ - CSharpExtArtifact('linux', 'android'), + CSharpExtArtifact('linux', 'android', arch_abi='arm64-v8a'), + CSharpExtArtifact('linux', 'android', arch_abi='armeabi-v7a'), PythonArtifact('linux', 'x86', 'cp27-cp27m'), PythonArtifact('linux', 'x86', 'cp27-cp27mu'), PythonArtifact('linux', 'x86', 'cp34-cp34m'), diff --git a/tools/run_tests/artifacts/build_artifact_csharp_android.sh b/tools/run_tests/artifacts/build_artifact_csharp_android.sh index e4f229e16a5..067eb30edcf 100755 --- a/tools/run_tests/artifacts/build_artifact_csharp_android.sh +++ b/tools/run_tests/artifacts/build_artifact_csharp_android.sh @@ -17,7 +17,7 @@ set -ex cd "$(dirname "$0")/../../.." -export ANDROID_ABI=armeabi-v7a +# ANDROID_ABI is set by the job definition in artifact_targets.py src/csharp/experimental/build_native_ext_for_android.sh mkdir -p "${ARTIFACTS_OUT}" From 8ed83fb44d6cf1f8a7c8e81b17c83bbacf33b185 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 20 Jul 2018 00:12:48 +0200 Subject: [PATCH 18/61] fix continuous build of the Grpc.Core nuget --- src/csharp/Grpc.Core/Grpc.Core.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/csharp/Grpc.Core/Grpc.Core.csproj b/src/csharp/Grpc.Core/Grpc.Core.csproj index 9e5eab10328..0da95d203c6 100755 --- a/src/csharp/Grpc.Core/Grpc.Core.csproj +++ b/src/csharp/Grpc.Core/Grpc.Core.csproj @@ -46,11 +46,11 @@ runtimes/win/native/grpc_csharp_ext.x86.dll true - + runtimes/monoandroid/armeabi-v7a/libgrpc_csharp_ext.so true - + runtimes/monoandroid/arm64-v8a/libgrpc_csharp_ext.so true From b47214952bafad402b6854f20bb546d828bf5573 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Thu, 19 Jul 2018 15:19:43 -0700 Subject: [PATCH 19/61] reviewer feedback --- include/grpcpp/ext/channelz_service_plugin.h | 20 +--------------- .../channelz/channelz_service_plugin.cc | 24 ++++++++++++------- test/cpp/end2end/channelz_service_test.cc | 6 ++--- 3 files changed, 20 insertions(+), 30 deletions(-) diff --git a/include/grpcpp/ext/channelz_service_plugin.h b/include/grpcpp/ext/channelz_service_plugin.h index d01372b0651..4a748a98800 100644 --- a/include/grpcpp/ext/channelz_service_plugin.h +++ b/include/grpcpp/ext/channelz_service_plugin.h @@ -25,31 +25,13 @@ #include #include -#include "src/cpp/server/channelz/channelz_service.h" - namespace grpc { namespace channelz { namespace experimental { -// This plugin is experimental for now. Track progress in -// https://github.com/grpc/grpc/issues/15988. -class ChannelzServicePlugin : public ::grpc::ServerBuilderPlugin { - public: - ChannelzServicePlugin(); - ::grpc::string name() override; - void InitServer(::grpc::ServerInitializer* si) override; - void Finish(::grpc::ServerInitializer* si) override; - void ChangeArguments(const ::grpc::string& name, void* value) override; - bool has_async_methods() const override; - bool has_sync_methods() const override; - - private: - std::shared_ptr channelz_service_; -}; - /// Add channelz server plugin to \a ServerBuilder. This function should /// be called at static initialization time. -void InitChannelzServerBuilderPlugin(); +void InitChannelzServiceBuilderPlugin(); } // namespace experimental } // namespace channelz diff --git a/src/cpp/server/channelz/channelz_service_plugin.cc b/src/cpp/server/channelz/channelz_service_plugin.cc index 8c2972915c7..c1c41bb57c8 100644 --- a/src/cpp/server/channelz/channelz_service_plugin.cc +++ b/src/cpp/server/channelz/channelz_service_plugin.cc @@ -29,6 +29,22 @@ namespace grpc { namespace channelz { namespace experimental { +// This plugin is experimental for now. Track progress in +// https://github.com/grpc/grpc/issues/15988. +class ChannelzServicePlugin : public ::grpc::ServerBuilderPlugin { + public: + ChannelzServicePlugin(); + ::grpc::string name() override; + void InitServer(::grpc::ServerInitializer* si) override; + void Finish(::grpc::ServerInitializer* si) override; + void ChangeArguments(const ::grpc::string& name, void* value) override; + bool has_async_methods() const override; + bool has_sync_methods() const override; + + private: + std::shared_ptr channelz_service_; +}; + ChannelzServicePlugin::ChannelzServicePlugin() : channelz_service_(new grpc::ChannelzService()) {} @@ -70,14 +86,6 @@ void InitChannelzServiceBuilderPlugin() { ::grpc::ServerBuilder::InternalAddPluginFactory(&CreateChannelzServicePlugin); } -// Force InitChannelzServiceBuilderPlugin() to be called at static -// initialization time. -struct StaticChannelServicePluginInitializer { - StaticChannelServicePluginInitializer() { - InitChannelzServiceBuilderPlugin(); - } -} static_channelz_service_plugin_initializer; - } // namespace experimental } // namespace channelz } // namespace grpc diff --git a/test/cpp/end2end/channelz_service_test.cc b/test/cpp/end2end/channelz_service_test.cc index 02d36795fc8..d52324d7b36 100644 --- a/test/cpp/end2end/channelz_service_test.cc +++ b/test/cpp/end2end/channelz_service_test.cc @@ -76,6 +76,9 @@ class ChannelzServerTest : public ::testing::Test { ChannelzServerTest() {} void SetUp() override { + // ensure channel server is brought up on all severs we build. + ::grpc::channelz::experimental::InitChannelzServiceBuilderPlugin(); + // We set up a proxy server with channelz enabled. proxy_port_ = grpc_pick_unused_port_or_die(); ServerBuilder proxy_builder; @@ -177,9 +180,6 @@ class ChannelzServerTest : public ::testing::Test { // backends. All implement the echo service. std::vector backends_; - - // ensure channel server is linked in. - channelz::experimental::ChannelzServicePlugin plugin_; }; TEST_F(ChannelzServerTest, BasicTest) { From 542ad27cf78bd95af8c34b0264433c4d7c0cb077 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 20 Jul 2018 01:22:53 +0200 Subject: [PATCH 20/61] run yapf --- tools/run_tests/artifacts/artifact_targets.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index ada7129bfc4..4500b220e97 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -219,8 +219,8 @@ class CSharpExtArtifact: self.arch_abi = arch_abi self.labels = ['artifact', 'csharp', platform, arch] if arch_abi: - self.name += '_%s' % arch_abi - self.labels.append(arch_abi) + self.name += '_%s' % arch_abi + self.labels.append(arch_abi) def pre_build_jobspecs(self): return [] @@ -231,7 +231,9 @@ class CSharpExtArtifact: self.name, 'tools/dockerfile/grpc_artifact_android_ndk', 'tools/run_tests/artifacts/build_artifact_csharp_android.sh', - environ={'ANDROID_ABI': self.arch_abi}) + environ={ + 'ANDROID_ABI': self.arch_abi + }) elif self.platform == 'windows': cmake_arch_option = 'Win32' if self.arch == 'x86' else self.arch return create_jobspec( From ad371f802d35686ad6776cc01d1287275f6dd9d6 Mon Sep 17 00:00:00 2001 From: Brent Shaffer Date: Thu, 19 Jul 2018 16:52:46 -0700 Subject: [PATCH 21/61] fixes subpackage issue with loading composer --- src/php/lib/Grpc/BaseStub.php | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/php/lib/Grpc/BaseStub.php b/src/php/lib/Grpc/BaseStub.php index ecb419ac8f8..fe81e377610 100644 --- a/src/php/lib/Grpc/BaseStub.php +++ b/src/php/lib/Grpc/BaseStub.php @@ -83,10 +83,11 @@ class BaseStub } private static function updateOpts($opts) { - $package_config = json_decode( - file_get_contents(dirname(__FILE__).'/../../composer.json'), - true - ); + if (!file_exists($composerFile = __DIR__.'/../../composer.json')) { + // for grpc/grpc-php subpackage + $composerFile = __DIR__.'/../composer.json'; + } + $package_config = json_decode(file_get_contents($composerFile), true); if (!empty($opts['grpc.primary_user_agent'])) { $opts['grpc.primary_user_agent'] .= ' '; } else { From 687316c025ef28a3006b1b65b0a2b9b5bbe692ad Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Thu, 19 Jul 2018 16:55:35 -0700 Subject: [PATCH 22/61] Clarify it's Objective-C sizes in comment --- tools/profiling/ios_bin/binary_size.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/profiling/ios_bin/binary_size.py b/tools/profiling/ios_bin/binary_size.py index cde09023f20..b07adb57345 100755 --- a/tools/profiling/ios_bin/binary_size.py +++ b/tools/profiling/ios_bin/binary_size.py @@ -86,7 +86,7 @@ def build(where, frameworks): 'src/objective-c/examples/Sample/Build-%s' % where) -text = '' +text = 'Objective-C binary sizes\n' for frameworks in [False, True]: build('new', frameworks) new_size = get_size('new', frameworks) From 18a9d7d57c44e071a722b399c27758786542cf56 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Fri, 20 Jul 2018 09:29:05 -0700 Subject: [PATCH 23/61] reviewer feedback --- BUILD | 4 +- include/grpcpp/ext/channelz_service_plugin.h | 5 +- .../channelz/channelz_service_plugin.cc | 56 ++++++++----------- test/cpp/end2end/channelz_service_test.cc | 2 +- 4 files changed, 29 insertions(+), 38 deletions(-) diff --git a/BUILD b/BUILD index 8bb98862e33..1c80e83ccae 100644 --- a/BUILD +++ b/BUILD @@ -2061,9 +2061,11 @@ grpc_cc_library( ], hdrs = [ "src/cpp/server/channelz/channelz_service.h", - "src/cpp/server/channelz/channelz_service_plugin.h", ], language = "c++", + public_hdrs = [ + "include/grpcpp/ext/channelz_service_plugin.h", + ], deps = [ ":grpc++", "//src/proto/grpc/channelz:channelz_proto", diff --git a/include/grpcpp/ext/channelz_service_plugin.h b/include/grpcpp/ext/channelz_service_plugin.h index 4a748a98800..af3192d4513 100644 --- a/include/grpcpp/ext/channelz_service_plugin.h +++ b/include/grpcpp/ext/channelz_service_plugin.h @@ -30,8 +30,9 @@ namespace channelz { namespace experimental { /// Add channelz server plugin to \a ServerBuilder. This function should -/// be called at static initialization time. -void InitChannelzServiceBuilderPlugin(); +/// be called at static initialization time. This service is experimental +/// for now. Track progress in https://github.com/grpc/grpc/issues/15988. +void InitChannelzService(); } // namespace experimental } // namespace channelz diff --git a/src/cpp/server/channelz/channelz_service_plugin.cc b/src/cpp/server/channelz/channelz_service_plugin.cc index c1c41bb57c8..b93e5b551e1 100644 --- a/src/cpp/server/channelz/channelz_service_plugin.cc +++ b/src/cpp/server/channelz/channelz_service_plugin.cc @@ -29,49 +29,37 @@ namespace grpc { namespace channelz { namespace experimental { -// This plugin is experimental for now. Track progress in -// https://github.com/grpc/grpc/issues/15988. class ChannelzServicePlugin : public ::grpc::ServerBuilderPlugin { public: - ChannelzServicePlugin(); - ::grpc::string name() override; - void InitServer(::grpc::ServerInitializer* si) override; - void Finish(::grpc::ServerInitializer* si) override; - void ChangeArguments(const ::grpc::string& name, void* value) override; - bool has_async_methods() const override; - bool has_sync_methods() const override; + ChannelzServicePlugin() : channelz_service_(new grpc::ChannelzService()) {} - private: - std::shared_ptr channelz_service_; -}; - -ChannelzServicePlugin::ChannelzServicePlugin() - : channelz_service_(new grpc::ChannelzService()) {} + grpc::string name() override { return "channelz_service"; } -grpc::string ChannelzServicePlugin::name() { return "channelz_service"; } - -void ChannelzServicePlugin::InitServer(grpc::ServerInitializer* si) { - si->RegisterService(channelz_service_); -} + void InitServer(grpc::ServerInitializer* si) override { + si->RegisterService(channelz_service_); + } -void ChannelzServicePlugin::Finish(grpc::ServerInitializer* si) {} + void Finish(grpc::ServerInitializer* si) override {} -void ChannelzServicePlugin::ChangeArguments(const grpc::string& name, - void* value) {} + void ChangeArguments(const grpc::string& name, void* value) override {} -bool ChannelzServicePlugin::has_sync_methods() const { - if (channelz_service_) { - return channelz_service_->has_synchronous_methods(); + bool has_sync_methods() const override { + if (channelz_service_) { + return channelz_service_->has_synchronous_methods(); + } + return false; } - return false; -} -bool ChannelzServicePlugin::has_async_methods() const { - if (channelz_service_) { - return channelz_service_->has_async_methods(); + bool has_async_methods() const override { + if (channelz_service_) { + return channelz_service_->has_async_methods(); + } + return false; } - return false; -} + + private: + std::shared_ptr channelz_service_; +}; static std::unique_ptr< ::grpc::ServerBuilderPlugin> CreateChannelzServicePlugin() { @@ -79,7 +67,7 @@ CreateChannelzServicePlugin() { new ChannelzServicePlugin()); } -void InitChannelzServiceBuilderPlugin() { +void InitChannelzService() { static bool already_here = false; if (already_here) return; already_here = true; diff --git a/test/cpp/end2end/channelz_service_test.cc b/test/cpp/end2end/channelz_service_test.cc index d52324d7b36..933e4a1ff67 100644 --- a/test/cpp/end2end/channelz_service_test.cc +++ b/test/cpp/end2end/channelz_service_test.cc @@ -77,7 +77,7 @@ class ChannelzServerTest : public ::testing::Test { void SetUp() override { // ensure channel server is brought up on all severs we build. - ::grpc::channelz::experimental::InitChannelzServiceBuilderPlugin(); + ::grpc::channelz::experimental::InitChannelzService(); // We set up a proxy server with channelz enabled. proxy_port_ = grpc_pick_unused_port_or_die(); From 746a80ea025359d372528c1b7e8a9decc356e593 Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Fri, 20 Jul 2018 11:37:26 -0700 Subject: [PATCH 24/61] Eliminate GOOGLE_APPLICATION_CREDENTIALS explicit setting --- tools/internal_ci/linux/grpc_publish_packages.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/internal_ci/linux/grpc_publish_packages.sh b/tools/internal_ci/linux/grpc_publish_packages.sh index 89da36987e0..d237d9fcdb1 100644 --- a/tools/internal_ci/linux/grpc_publish_packages.sh +++ b/tools/internal_ci/linux/grpc_publish_packages.sh @@ -17,8 +17,6 @@ set -ex shopt -s nullglob -export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json - GCS_ROOT=gs://packages.grpc.io MANIFEST_FILE=index.xml ARCHIVE_UUID=${KOKORO_BUILD_ID:-$(uuidgen)} From 0c798357dae692e8ba644258c9432c87b0b20b2a Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Fri, 20 Jul 2018 11:37:49 -0700 Subject: [PATCH 25/61] Fix GIT_BRANCH_NAME to master for now --- tools/internal_ci/linux/grpc_publish_packages.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/internal_ci/linux/grpc_publish_packages.sh b/tools/internal_ci/linux/grpc_publish_packages.sh index d237d9fcdb1..5ad14931259 100644 --- a/tools/internal_ci/linux/grpc_publish_packages.sh +++ b/tools/internal_ci/linux/grpc_publish_packages.sh @@ -20,7 +20,7 @@ shopt -s nullglob GCS_ROOT=gs://packages.grpc.io MANIFEST_FILE=index.xml ARCHIVE_UUID=${KOKORO_BUILD_ID:-$(uuidgen)} -GIT_BRANCH_NAME=master #${KOKORO_GITHUB_COMMIT:-master} +GIT_BRANCH_NAME=master GIT_COMMIT=${KOKORO_GIT_COMMIT:-unknown} ARCHIVE_TIMESTAMP=$(date -Iseconds) TARGET_DIR=$(mktemp -d grpc_publish_packages.sh.XXXX) From 4b63744d0caf48c256339926d16eb373917478ec Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Fri, 20 Jul 2018 12:39:34 -0700 Subject: [PATCH 26/61] Refactor and enhance the publish package script --- .../linux/grpc_publish_packages.sh | 160 +++++++++++------- 1 file changed, 98 insertions(+), 62 deletions(-) diff --git a/tools/internal_ci/linux/grpc_publish_packages.sh b/tools/internal_ci/linux/grpc_publish_packages.sh index 5ad14931259..24bb4880ed8 100644 --- a/tools/internal_ci/linux/grpc_publish_packages.sh +++ b/tools/internal_ci/linux/grpc_publish_packages.sh @@ -17,92 +17,128 @@ set -ex shopt -s nullglob -GCS_ROOT=gs://packages.grpc.io -MANIFEST_FILE=index.xml -ARCHIVE_UUID=${KOKORO_BUILD_ID:-$(uuidgen)} -GIT_BRANCH_NAME=master -GIT_COMMIT=${KOKORO_GIT_COMMIT:-unknown} -ARCHIVE_TIMESTAMP=$(date -Iseconds) -TARGET_DIR=$(mktemp -d grpc_publish_packages.sh.XXXX) -YEAR_MONTH_PREFIX=$(date "+%Y/%m") -YEAR_PREFIX=${YEAR_MONTH_PREFIX%%/*} -UPLOAD_ROOT=$TARGET_DIR/$YEAR_PREFIX -RELATIVE_PATH=$YEAR_MONTH_PREFIX/$ARCHIVE_UUID -BUILD_ROOT=$TARGET_DIR/$RELATIVE_PATH - -LINUX_PACKAGES=$KOKORO_GFILE_DIR/github/grpc/artifacts -WINDOWS_PACKAGES=$KOKORO_GFILE_DIR/github/grpc/artifacts -# TODO(mmx): enable linux_extra -# LINUX_EXTRA_PACKAGES=$KOKORO_GFILE_DIR/github/grpc/artifacts +INPUT_ARTIFACTS=$KOKORO_GFILE_DIR/github/grpc/artifacts +INDEX_FILENAME=index.xml + +BUILD_ID=${KOKORO_BUILD_ID:-$(uuidgen)} +BUILD_BRANCH_NAME=master +BUILD_GIT_COMMIT=${KOKORO_GIT_COMMIT:-unknown} +BUILD_TIMESTAMP=$(date -Iseconds) +BUILD_RELPATH=$(date "+%Y/%m")/$BUILD_ID/ + +GCS_ROOT=gs://packages.grpc.io/ +GCS_ARCHIVE_PREFIX=archive/ +GCS_ARCHIVE_ROOT=$GCS_ROOT$GCS_ARCHIVE_PREFIX +GCS_INDEX=$GCS_ROOT$INDEX_FILENAME + +LOCAL_STAGING_TEMPDIR=$(mktemp -d) +LOCAL_BUILD_ROOT=$LOCAL_STAGING_TEMPDIR/$BUILD_RELPATH +LOCAL_BUILD_INDEX=$LOCAL_BUILD_ROOT$INDEX_FILENAME + +mkdir -p "$LOCAL_BUILD_ROOT" + +find "$INPUT_ARTIFACTS" -type f + +UNZIPPED_CSHARP_PACKAGES=$(mktemp -d) +unzip "$INPUT_ARTIFACTS/csharp_nugets_windows_dotnetcli.zip" -d "$UNZIPPED_CSHARP_PACKAGES" +CSHARP_PACKAGES=( + "$UNZIPPED_CSHARP_PACKAGES"/* +) PYTHON_PACKAGES=( - "$LINUX_PACKAGES"/grpcio-[0-9]*.whl - "$LINUX_PACKAGES"/grpcio-[0-9]*.tar.gz - "$LINUX_PACKAGES"/grpcio_tools-[0-9]*.whl - "$LINUX_PACKAGES"/grpcio-tools-[0-9]*.tar.gz - "$LINUX_PACKAGES"/grpcio-health-checking-[0-9]*.tar.gz - "$LINUX_PACKAGES"/grpcio-reflection-[0-9]*.tar.gz - "$LINUX_PACKAGES"/grpcio-testing-[0-9]*.tar.gz - #"$LINUX_EXTRA_PACKAGES"/grpcio-[0-9]*.whl - #"$LINUX_EXTRA_PACKAGES"/grpcio_tools-[0-9]*.whl + "$INPUT_ARTIFACTS"/grpcio-[0-9]*.tar.gz + "$INPUT_ARTIFACTS"/grpcio-[0-9]*.whl + "$INPUT_ARTIFACTS"/python_linux_extra_arm*/grpcio-[0-9]*.whl + + "$INPUT_ARTIFACTS"/grpcio-tools-[0-9]*.tar.gz + "$INPUT_ARTIFACTS"/grpcio_tools-[0-9]*.whl + "$INPUT_ARTIFACTS"/python_linux_extra_arm*/grpcio_tools-[0-9]*.whl + + "$INPUT_ARTIFACTS"/grpcio-health-checking-[0-9]*.tar.gz + "$INPUT_ARTIFACTS"/grpcio-reflection-[0-9]*.tar.gz + "$INPUT_ARTIFACTS"/grpcio-testing-[0-9]*.tar.gz ) PHP_PACKAGES=( - "$LINUX_PACKAGES"/grpc-[0-9]*.tgz + "$INPUT_ARTIFACTS"/grpc-[0-9]*.tgz ) RUBY_PACKAGES=( - "$LINUX_PACKAGES"/grpc-[0-9]*.gem - "$LINUX_PACKAGES"/grpc-tools-[0-9]*.gem -) - -CSHARP_PACKAGES=( - "$WINDOWS_PACKAGES"/csharp_nugets_windows_dotnetcli.zip + "$INPUT_ARTIFACTS"/grpc-[0-9]*.gem + "$INPUT_ARTIFACTS"/grpc-tools-[0-9]*.gem ) function add_to_manifest() { - local xml_type=$1 - local xml_name - xml_name=$(basename "$2") - local xml_sha256 - xml_sha256=$(openssl sha256 -r "$2" | cut -d " " -f 1) - cp "$2" "$BUILD_ROOT" - echo "" + local artifact_type=$1 + local artifact_file=$2 + local artifact_name + artifact_name=$(basename "$artifact_file") + local artifact_sha256 + artifact_sha256=$(openssl sha256 -r "$artifact_file" | cut -d " " -f 1) + local artifact_target=$LOCAL_BUILD_ROOT/$artifact_type + mkdir -p "$artifact_target" + cp "$artifact_file" "$artifact_target" + cat < +EOF } -mkdir -p "$BUILD_ROOT" - { cat < - + + + + gRPC + https://github.com/grpc/grpc + $BUILD_BRANCH_NAME + $BUILD_GIT_COMMIT + + EOF - echo "" - echo "" - echo "$GIT_BRANCH_NAME" - echo "$GIT_COMMIT" - echo "" - for pkg in "${PYTHON_PACKAGES[@]}"; do add_to_manifest python "$pkg"; done for pkg in "${CSHARP_PACKAGES[@]}"; do add_to_manifest csharp "$pkg"; done for pkg in "${PHP_PACKAGES[@]}"; do add_to_manifest php "$pkg"; done + for pkg in "${PYTHON_PACKAGES[@]}"; do add_to_manifest python "$pkg"; done for pkg in "${RUBY_PACKAGES[@]}"; do add_to_manifest ruby "$pkg"; done - echo "" -}> "$BUILD_ROOT/$MANIFEST_FILE" + cat < + +EOF +}> "$LOCAL_BUILD_INDEX" + +LOCAL_BUILD_INDEX_SHA256=$(openssl sha256 -r "$LOCAL_BUILD_INDEX" | cut -d " " -f 1) -BUILD_XML_SHA=$(openssl sha256 -r "$BUILD_ROOT/$MANIFEST_FILE" | cut -d " " -f 1) +OLD_INDEX=$(mktemp) +NEW_INDEX=$(mktemp) -PREV_HOME=$(mktemp old-XXXXX-$MANIFEST_FILE) -NEW_HOME=$(mktemp new-XXXXX-$MANIFEST_FILE) -gsutil cp "$GCS_ROOT/$MANIFEST_FILE" "$PREV_HOME" +# Download the current /index.xml into $OLD_INDEX +gsutil cp "$GCS_INDEX" "$OLD_INDEX" { - head --lines=4 "$PREV_HOME" - echo "" - tail --lines=+5 "$PREV_HOME" -}> "$NEW_HOME" - -gsutil -m cp -r "$UPLOAD_ROOT" "$GCS_ROOT/archive" -gsutil -h "Content-Type:application/xml" cp "$NEW_HOME" "$GCS_ROOT/$MANIFEST_FILE" + # we want to add an entry as the first child under tag + # we can get by without a real XML parser by rewriting the header, + # injecting our new tag, and then dumping the rest of the file as is. + cat < + + + + +EOF + tail --lines=+5 "$OLD_INDEX" +}> "$NEW_INDEX" +# Upload the current build artifacts +gsutil -m cp -r "$LOCAL_STAGING_TEMPDIR/${BUILD_RELPATH%%/*}" "$GCS_ARCHIVE_ROOT" +# Upload the new /index.xml +gsutil -h "Content-Type:application/xml" cp "$NEW_INDEX" "$GCS_INDEX" From a9bfe94d10b0f49b6d75149276597669aa595cb6 Mon Sep 17 00:00:00 2001 From: Juanli Shen Date: Sun, 22 Jul 2018 16:36:13 -0700 Subject: [PATCH 27/61] Relax qps/eps error in load reporter test --- test/cpp/server/load_reporter/load_reporter_test.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/cpp/server/load_reporter/load_reporter_test.cc b/test/cpp/server/load_reporter/load_reporter_test.cc index 719c3a67d96..0d56cdf431d 100644 --- a/test/cpp/server/load_reporter/load_reporter_test.cc +++ b/test/cpp/server/load_reporter/load_reporter_test.cc @@ -172,9 +172,9 @@ class LbFeedbackTest : public LoadReporterTest { // TODO(juanlishen): The error is big because we use sleep(). It should be // much smaller when we use fake clock. ASSERT_THAT(static_cast(lb_feedback.calls_per_second()), - DoubleNear(expected_qps, expected_qps / 50)); + DoubleNear(expected_qps, expected_qps * 0.05)); ASSERT_THAT(static_cast(lb_feedback.errors_per_second()), - DoubleNear(expected_eps, expected_eps / 50)); + DoubleNear(expected_eps, expected_eps * 0.05)); gpr_log(GPR_INFO, "Verified LB feedback matches the samples of index [%lu, %lu).", start, start + count); From 1add55dbda076393bea0419b707f3353235d05ff Mon Sep 17 00:00:00 2001 From: Matthew Leibowitz Date: Sat, 21 Jul 2018 19:40:19 +0200 Subject: [PATCH 28/61] iOS (and probably other AOT platforms) needs to have delegates registered - if a managed delegate is going to be passed to native code, then it requires an attribute - instead of depending on Xamarin.iOS for this, we can just create our own, the iOS runtime just checks for the type name - reference: https://docs.microsoft.com/en-gb/xamarin/ios/internals/limitations#reverse-callbacks --- src/csharp/Grpc.Core/Internal/NativeLogRedirector.cs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/csharp/Grpc.Core/Internal/NativeLogRedirector.cs b/src/csharp/Grpc.Core/Internal/NativeLogRedirector.cs index bf6440123a6..72dd21dc946 100644 --- a/src/csharp/Grpc.Core/Internal/NativeLogRedirector.cs +++ b/src/csharp/Grpc.Core/Internal/NativeLogRedirector.cs @@ -51,6 +51,7 @@ namespace Grpc.Core.Internal } } + [MonoPInvokeCallback(typeof(GprLogDelegate))] private static void HandleWrite(IntPtr fileStringPtr, int line, ulong threadId, IntPtr severityStringPtr, IntPtr msgPtr) { try @@ -86,4 +87,15 @@ namespace Grpc.Core.Internal } } } + + [AttributeUsage(AttributeTargets.Method)] + internal sealed class MonoPInvokeCallbackAttribute : Attribute + { + public MonoPInvokeCallbackAttribute(Type type) + { + Type = type; + } + + public Type Type { get; private set; } + } } From bf5524b1c1920242e7b04aedf466da580733c864 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 23 Jul 2018 18:01:17 +0200 Subject: [PATCH 29/61] add explanation for MonoPInvokeCallback attribute --- src/csharp/Grpc.Core/Internal/NativeLogRedirector.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/csharp/Grpc.Core/Internal/NativeLogRedirector.cs b/src/csharp/Grpc.Core/Internal/NativeLogRedirector.cs index 72dd21dc946..30264acb10b 100644 --- a/src/csharp/Grpc.Core/Internal/NativeLogRedirector.cs +++ b/src/csharp/Grpc.Core/Internal/NativeLogRedirector.cs @@ -88,6 +88,13 @@ namespace Grpc.Core.Internal } } + /// + /// Use this attribute to mark methods that will be called back from P/Invoke calls. + /// iOS (and probably other AOT platforms) needs to have delegates registered. + /// Instead of depending on Xamarin.iOS for this, we can just create our own, + /// the iOS runtime just checks for the type name. + /// See: https://docs.microsoft.com/en-gb/xamarin/ios/internals/limitations#reverse-callbacks + /// [AttributeUsage(AttributeTargets.Method)] internal sealed class MonoPInvokeCallbackAttribute : Attribute { From 632eb7acaac3ef55ddd9256a68bfabce5157e6ab Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Mon, 23 Jul 2018 08:40:06 -0700 Subject: [PATCH 30/61] Add protoc packages to nightly builds --- .../linux/grpc_publish_packages.sh | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tools/internal_ci/linux/grpc_publish_packages.sh b/tools/internal_ci/linux/grpc_publish_packages.sh index 24bb4880ed8..ebd19c59542 100644 --- a/tools/internal_ci/linux/grpc_publish_packages.sh +++ b/tools/internal_ci/linux/grpc_publish_packages.sh @@ -39,12 +39,31 @@ mkdir -p "$LOCAL_BUILD_ROOT" find "$INPUT_ARTIFACTS" -type f +# protoc Plugins +PROTOC_PLUGINS_ZIPPED_PACKAGES=$(mktemp -d) +for zip_dir in protoc_windows_{x86,x64} +do + zip -jr "$PROTOC_PLUGINS_ZIPPED_PACKAGES/$zip_dir.zip" "$INPUT_ARTIFACTS/$zip_dir/"* +done +for tar_dir in protoc_{linux,macos}_{x86,x64} +do + chmod +x "$INPUT_ARTIFACTS/$tar_dir"/* + tar -cvzf "$PROTOC_PLUGINS_ZIPPED_PACKAGES/$tar_dir.tar.gz" -C "$INPUT_ARTIFACTS/$tar_dir" . +done + +PROTOC_PACKAGES=( + "$PROTOC_PLUGINS_ZIPPED_PACKAGES"/protoc_windows_{x86,x64}.zip + "$PROTOC_PLUGINS_ZIPPED_PACKAGES"/protoc_{linux,macos}_{x86,x64}.tar.gz +) + +# C# UNZIPPED_CSHARP_PACKAGES=$(mktemp -d) unzip "$INPUT_ARTIFACTS/csharp_nugets_windows_dotnetcli.zip" -d "$UNZIPPED_CSHARP_PACKAGES" CSHARP_PACKAGES=( "$UNZIPPED_CSHARP_PACKAGES"/* ) +# Python PYTHON_PACKAGES=( "$INPUT_ARTIFACTS"/grpcio-[0-9]*.tar.gz "$INPUT_ARTIFACTS"/grpcio-[0-9]*.whl @@ -59,10 +78,12 @@ PYTHON_PACKAGES=( "$INPUT_ARTIFACTS"/grpcio-testing-[0-9]*.tar.gz ) +# PHP PHP_PACKAGES=( "$INPUT_ARTIFACTS"/grpc-[0-9]*.tgz ) +# Ruby RUBY_PACKAGES=( "$INPUT_ARTIFACTS"/grpc-[0-9]*.gem "$INPUT_ARTIFACTS"/grpc-tools-[0-9]*.gem @@ -100,6 +121,7 @@ EOF EOF + for pkg in "${PROTOC_PACKAGES[@]}"; do add_to_manifest protoc "$pkg"; done for pkg in "${CSHARP_PACKAGES[@]}"; do add_to_manifest csharp "$pkg"; done for pkg in "${PHP_PACKAGES[@]}"; do add_to_manifest php "$pkg"; done for pkg in "${PYTHON_PACKAGES[@]}"; do add_to_manifest python "$pkg"; done From 536c4aadf3f5ebb84281c6f1ed05d3e649aac18f Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Mon, 23 Jul 2018 08:44:43 -0700 Subject: [PATCH 31/61] Add template version number to build manifest XML files --- tools/internal_ci/linux/grpc_publish_packages.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/internal_ci/linux/grpc_publish_packages.sh b/tools/internal_ci/linux/grpc_publish_packages.sh index ebd19c59542..2439e40c7e5 100644 --- a/tools/internal_ci/linux/grpc_publish_packages.sh +++ b/tools/internal_ci/linux/grpc_publish_packages.sh @@ -111,7 +111,7 @@ EOF cat < - + gRPC https://github.com/grpc/grpc From 775f31e1746278ae457d496cefc6ea8323ad855f Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Mon, 23 Jul 2018 09:18:04 -0700 Subject: [PATCH 32/61] Put different Python packages in different directories --- .../linux/grpc_publish_packages.sh | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/tools/internal_ci/linux/grpc_publish_packages.sh b/tools/internal_ci/linux/grpc_publish_packages.sh index 2439e40c7e5..cab5cf05146 100644 --- a/tools/internal_ci/linux/grpc_publish_packages.sh +++ b/tools/internal_ci/linux/grpc_publish_packages.sh @@ -64,17 +64,23 @@ CSHARP_PACKAGES=( ) # Python -PYTHON_PACKAGES=( +PYTHON_GRPCIO_PACKAGES=( "$INPUT_ARTIFACTS"/grpcio-[0-9]*.tar.gz "$INPUT_ARTIFACTS"/grpcio-[0-9]*.whl "$INPUT_ARTIFACTS"/python_linux_extra_arm*/grpcio-[0-9]*.whl - +) +PYTHON_GRPCIO_TOOLS_PACKAGES=( "$INPUT_ARTIFACTS"/grpcio-tools-[0-9]*.tar.gz "$INPUT_ARTIFACTS"/grpcio_tools-[0-9]*.whl "$INPUT_ARTIFACTS"/python_linux_extra_arm*/grpcio_tools-[0-9]*.whl - +) +PYTHON_GRPCIO_HEALTH_CHECKING_PACKAGES=( "$INPUT_ARTIFACTS"/grpcio-health-checking-[0-9]*.tar.gz +) +PYTHON_GRPCIO_REFLECTION_PACKAGES=( "$INPUT_ARTIFACTS"/grpcio-reflection-[0-9]*.tar.gz +) +PYTHON_GRPCIO_TESTING_PACKAGES=( "$INPUT_ARTIFACTS"/grpcio-testing-[0-9]*.tar.gz ) @@ -92,17 +98,18 @@ RUBY_PACKAGES=( function add_to_manifest() { local artifact_type=$1 local artifact_file=$2 + local artifact_prefix=$3 local artifact_name artifact_name=$(basename "$artifact_file") local artifact_sha256 artifact_sha256=$(openssl sha256 -r "$artifact_file" | cut -d " " -f 1) - local artifact_target=$LOCAL_BUILD_ROOT/$artifact_type + local artifact_target=$LOCAL_BUILD_ROOT/$artifact_type/$artifact_prefix mkdir -p "$artifact_target" cp "$artifact_file" "$artifact_target" cat < EOF } @@ -124,7 +131,11 @@ EOF for pkg in "${PROTOC_PACKAGES[@]}"; do add_to_manifest protoc "$pkg"; done for pkg in "${CSHARP_PACKAGES[@]}"; do add_to_manifest csharp "$pkg"; done for pkg in "${PHP_PACKAGES[@]}"; do add_to_manifest php "$pkg"; done - for pkg in "${PYTHON_PACKAGES[@]}"; do add_to_manifest python "$pkg"; done + for pkg in "${PYTHON_GRPCIO_PACKAGES[@]}"; do add_to_manifest python "$pkg" grpcio/; done + for pkg in "${PYTHON_GRPCIO_TOOLS_PACKAGES[@]}"; do add_to_manifest python "$pkg" grpcio-tools/; done + for pkg in "${PYTHON_GRPCIO_HEALTH_CHECKING_PACKAGES[@]}"; do add_to_manifest python "$pkg" grpcio-health-checking/; done + for pkg in "${PYTHON_GRPCIO_REFLECTION_PACKAGES[@]}"; do add_to_manifest python "$pkg" grpcio-reflection/; done + for pkg in "${PYTHON_GRPCIO_TESTING_PACKAGES[@]}"; do add_to_manifest python "$pkg" grpcio-testing/; done for pkg in "${RUBY_PACKAGES[@]}"; do add_to_manifest ruby "$pkg"; done cat < Date: Mon, 23 Jul 2018 09:58:51 -0700 Subject: [PATCH 33/61] Add file size to manifest files --- tools/internal_ci/linux/grpc_publish_packages.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/internal_ci/linux/grpc_publish_packages.sh b/tools/internal_ci/linux/grpc_publish_packages.sh index cab5cf05146..bf0c313599b 100644 --- a/tools/internal_ci/linux/grpc_publish_packages.sh +++ b/tools/internal_ci/linux/grpc_publish_packages.sh @@ -101,6 +101,8 @@ function add_to_manifest() { local artifact_prefix=$3 local artifact_name artifact_name=$(basename "$artifact_file") + local artifact_size + artifact_size=$(stat -c%s "$artifact_file") local artifact_sha256 artifact_sha256=$(openssl sha256 -r "$artifact_file" | cut -d " " -f 1) local artifact_target=$LOCAL_BUILD_ROOT/$artifact_type/$artifact_prefix @@ -110,6 +112,7 @@ function add_to_manifest() { EOF } @@ -144,6 +147,7 @@ EOF EOF }> "$LOCAL_BUILD_INDEX" +LOCAL_BUILD_INDEX_SIZE=$(stat -c%s "$LOCAL_BUILD_INDEX") LOCAL_BUILD_INDEX_SHA256=$(openssl sha256 -r "$LOCAL_BUILD_INDEX" | cut -d " " -f 1) OLD_INDEX=$(mktemp) @@ -166,6 +170,7 @@ gsutil cp "$GCS_INDEX" "$OLD_INDEX" branch='$BUILD_BRANCH_NAME' commit='$BUILD_GIT_COMMIT' path='$GCS_ARCHIVE_PREFIX$BUILD_RELPATH$INDEX_FILENAME' + size='$LOCAL_BUILD_INDEX_SIZE' sha256='$LOCAL_BUILD_INDEX_SHA256' /> EOF tail --lines=+5 "$OLD_INDEX" From 92e0a70ec64f61918faab50a14b0587cf1aaaa3a Mon Sep 17 00:00:00 2001 From: Alex Polcyn Date: Sat, 21 Jul 2018 07:38:34 +0000 Subject: [PATCH 34/61] Make QPS server builder customizeabled --- CMakeLists.txt | 1 + Makefile | 3 ++ build.yaml | 2 + grpc.gyp | 1 + test/cpp/qps/BUILD | 6 +++ test/cpp/qps/qps_server_builder.cc | 45 ++++++++++++++++++ test/cpp/qps/qps_server_builder.h | 46 +++++++++++++++++++ test/cpp/qps/qps_worker.cc | 9 ++-- test/cpp/qps/server_async.cc | 15 +++--- test/cpp/qps/server_sync.cc | 13 +++--- .../generated/sources_and_headers.json | 3 ++ 11 files changed, 127 insertions(+), 17 deletions(-) create mode 100644 test/cpp/qps/qps_server_builder.cc create mode 100644 test/cpp/qps/qps_server_builder.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 75f3a2f3cdf..93991de4224 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5164,6 +5164,7 @@ add_library(qps test/cpp/qps/client_sync.cc test/cpp/qps/driver.cc test/cpp/qps/parse_json.cc + test/cpp/qps/qps_server_builder.cc test/cpp/qps/qps_worker.cc test/cpp/qps/report.cc test/cpp/qps/server_async.cc diff --git a/Makefile b/Makefile index 309187969cc..f829b3abd08 100644 --- a/Makefile +++ b/Makefile @@ -7438,6 +7438,7 @@ LIBQPS_SRC = \ test/cpp/qps/client_sync.cc \ test/cpp/qps/driver.cc \ test/cpp/qps/parse_json.cc \ + test/cpp/qps/qps_server_builder.cc \ test/cpp/qps/qps_worker.cc \ test/cpp/qps/report.cc \ test/cpp/qps/server_async.cc \ @@ -7493,6 +7494,7 @@ $(OBJDIR)/$(CONFIG)/test/cpp/qps/client_async.o: $(GENDIR)/src/proto/grpc/testin $(OBJDIR)/$(CONFIG)/test/cpp/qps/client_sync.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/qps/driver.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/qps/parse_json.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/qps/qps_server_builder.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/qps/qps_worker.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/qps/report.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/qps/server_async.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.grpc.pb.cc @@ -24691,6 +24693,7 @@ test/cpp/qps/client_async.cc: $(OPENSSL_DEP) test/cpp/qps/client_sync.cc: $(OPENSSL_DEP) test/cpp/qps/driver.cc: $(OPENSSL_DEP) test/cpp/qps/parse_json.cc: $(OPENSSL_DEP) +test/cpp/qps/qps_server_builder.cc: $(OPENSSL_DEP) test/cpp/qps/qps_worker.cc: $(OPENSSL_DEP) test/cpp/qps/report.cc: $(OPENSSL_DEP) test/cpp/qps/server_async.cc: $(OPENSSL_DEP) diff --git a/build.yaml b/build.yaml index 2ff98f12bc6..c76aa464354 100644 --- a/build.yaml +++ b/build.yaml @@ -1963,6 +1963,7 @@ libs: - test/cpp/qps/histogram.h - test/cpp/qps/interarrival.h - test/cpp/qps/parse_json.h + - test/cpp/qps/qps_server_builder.h - test/cpp/qps/qps_worker.h - test/cpp/qps/report.h - test/cpp/qps/server.h @@ -1981,6 +1982,7 @@ libs: - test/cpp/qps/client_sync.cc - test/cpp/qps/driver.cc - test/cpp/qps/parse_json.cc + - test/cpp/qps/qps_server_builder.cc - test/cpp/qps/qps_worker.cc - test/cpp/qps/report.cc - test/cpp/qps/server_async.cc diff --git a/grpc.gyp b/grpc.gyp index 7cbd9eded78..e1485efa058 100644 --- a/grpc.gyp +++ b/grpc.gyp @@ -1718,6 +1718,7 @@ 'test/cpp/qps/client_sync.cc', 'test/cpp/qps/driver.cc', 'test/cpp/qps/parse_json.cc', + 'test/cpp/qps/qps_server_builder.cc', 'test/cpp/qps/qps_worker.cc', 'test/cpp/qps/report.cc', 'test/cpp/qps/server_async.cc', diff --git a/test/cpp/qps/BUILD b/test/cpp/qps/BUILD index e7d093c71a4..b958c75fc7c 100644 --- a/test/cpp/qps/BUILD +++ b/test/cpp/qps/BUILD @@ -34,11 +34,13 @@ grpc_cc_library( "qps_worker.cc", "server_async.cc", "server_sync.cc", + "qps_server_builder.cc", ], hdrs = [ "client.h", "qps_worker.h", "server.h", + "qps_server_builder.h", ], deps = [ ":histogram", @@ -55,6 +57,10 @@ grpc_cc_library( "//test/core/util:gpr_test_util", "//test/core/util:grpc_test_util", "//test/cpp/util:test_util", + "//test/cpp/util:test_config", + ], + external_deps = [ + "gflags", ], ) diff --git a/test/cpp/qps/qps_server_builder.cc b/test/cpp/qps/qps_server_builder.cc new file mode 100644 index 00000000000..5fbc682b756 --- /dev/null +++ b/test/cpp/qps/qps_server_builder.cc @@ -0,0 +1,45 @@ +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "qps_server_builder.h" + +using grpc::ServerBuilder; + +namespace grpc { +namespace testing { + +namespace { +std::unique_ptr DefaultCreateQpsServerBuilder() { + return std::unique_ptr(new ServerBuilder()); +} + +std::function()> g_create_qps_server_builder = + DefaultCreateQpsServerBuilder; +} // namespace + +std::unique_ptr CreateQpsServerBuilder() { + return g_create_qps_server_builder(); +} + +void SetCreateQpsServerBuilderFunc( + std::function()> create_qps_server_builder) { + g_create_qps_server_builder = std::move(create_qps_server_builder); +} + +} // namespace testing +} // namespace grpc diff --git a/test/cpp/qps/qps_server_builder.h b/test/cpp/qps/qps_server_builder.h new file mode 100644 index 00000000000..98f9fa72e97 --- /dev/null +++ b/test/cpp/qps/qps_server_builder.h @@ -0,0 +1,46 @@ +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPC_QPS_SERVER_BUILDER_H +#define GRPC_QPS_SERVER_BUILDER_H + +#include +#include + +#include + +namespace grpc { +namespace testing { + +// CreateQpsServerBuilder creates a new ServerBuilder. +// This uses the "create ServerBuilder" func that was set +// in SetCreateQpsServerBuilderFunc if one has been set, +// otherwise, this defaults to creating a new ServerBuilder +// with only its default constructor. +std::unique_ptr CreateQpsServerBuilder(); + +// SetCreateQpsServerBuilderFunc sets a function to use to create new +// ServerBuilders in "CreateQpsServerBuilder". It can be used to modify options +// that the server is built with. +void SetCreateQpsServerBuilderFunc( + std::function()>); + +} // namespace testing +} // namespace grpc + +#endif // GRPC_QPS_SERVER_BUILDER_H diff --git a/test/cpp/qps/qps_worker.cc b/test/cpp/qps/qps_worker.cc index d3f03804743..7ddf3c1cf3b 100644 --- a/test/cpp/qps/qps_worker.cc +++ b/test/cpp/qps/qps_worker.cc @@ -39,6 +39,7 @@ #include "test/core/util/grpc_profiler.h" #include "test/core/util/histogram.h" #include "test/cpp/qps/client.h" +#include "test/cpp/qps/qps_server_builder.h" #include "test/cpp/qps/server.h" #include "test/cpp/util/create_test_channel.h" #include "test/cpp/util/test_credentials_provider.h" @@ -272,18 +273,18 @@ QpsWorker::QpsWorker(int driver_port, int server_port, impl_.reset(new WorkerServiceImpl(server_port, this)); gpr_atm_rel_store(&done_, static_cast(0)); - ServerBuilder builder; + std::unique_ptr builder = CreateQpsServerBuilder(); if (driver_port >= 0) { char* server_address = nullptr; gpr_join_host_port(&server_address, "::", driver_port); - builder.AddListeningPort( + builder->AddListeningPort( server_address, GetCredentialsProvider()->GetServerCredentials(credential_type)); gpr_free(server_address); } - builder.RegisterService(impl_.get()); + builder->RegisterService(impl_.get()); - server_ = builder.BuildAndStart(); + server_ = builder->BuildAndStart(); } QpsWorker::~QpsWorker() {} diff --git a/test/cpp/qps/server_async.cc b/test/cpp/qps/server_async.cc index 1dfef6cfc17..5cd975cf742 100644 --- a/test/cpp/qps/server_async.cc +++ b/test/cpp/qps/server_async.cc @@ -38,6 +38,7 @@ #include "src/core/lib/surface/completion_queue.h" #include "src/proto/grpc/testing/benchmark_service.grpc.pb.h" #include "test/core/util/test_config.h" +#include "test/cpp/qps/qps_server_builder.h" #include "test/cpp/qps/server.h" namespace grpc { @@ -74,19 +75,19 @@ class AsyncQpsServerTest final : public grpc::testing::Server { ResponseType*)> process_rpc) : Server(config) { - ServerBuilder builder; + std::unique_ptr builder = CreateQpsServerBuilder(); auto port_num = port(); // Negative port number means inproc server, so no listen port needed if (port_num >= 0) { char* server_address = nullptr; gpr_join_host_port(&server_address, "::", port_num); - builder.AddListeningPort(server_address, - Server::CreateServerCredentials(config)); + builder->AddListeningPort(server_address, + Server::CreateServerCredentials(config)); gpr_free(server_address); } - register_service(&builder, &async_service_); + register_service(builder.get(), &async_service_); int num_threads = config.async_server_threads(); if (num_threads <= 0) { // dynamic sizing @@ -97,15 +98,15 @@ class AsyncQpsServerTest final : public grpc::testing::Server { int tpc = std::max(1, config.threads_per_cq()); // 1 if unspecified int num_cqs = (num_threads + tpc - 1) / tpc; // ceiling operator for (int i = 0; i < num_cqs; i++) { - srv_cqs_.emplace_back(builder.AddCompletionQueue()); + srv_cqs_.emplace_back(builder->AddCompletionQueue()); } for (int i = 0; i < num_threads; i++) { cq_.emplace_back(i % srv_cqs_.size()); } - ApplyConfigToBuilder(config, &builder); + ApplyConfigToBuilder(config, builder.get()); - server_ = builder.BuildAndStart(); + server_ = builder->BuildAndStart(); auto process_rpc_bound = std::bind(process_rpc, config.payload_config(), std::placeholders::_1, diff --git a/test/cpp/qps/server_sync.cc b/test/cpp/qps/server_sync.cc index b8facf9b569..2e63f5ec867 100644 --- a/test/cpp/qps/server_sync.cc +++ b/test/cpp/qps/server_sync.cc @@ -27,6 +27,7 @@ #include "src/core/lib/gpr/host_port.h" #include "src/proto/grpc/testing/benchmark_service.grpc.pb.h" +#include "test/cpp/qps/qps_server_builder.h" #include "test/cpp/qps/server.h" #include "test/cpp/qps/usage_timer.h" @@ -154,23 +155,23 @@ class BenchmarkServiceImpl final : public BenchmarkService::Service { class SynchronousServer final : public grpc::testing::Server { public: explicit SynchronousServer(const ServerConfig& config) : Server(config) { - ServerBuilder builder; + std::unique_ptr builder = CreateQpsServerBuilder(); auto port_num = port(); // Negative port number means inproc server, so no listen port needed if (port_num >= 0) { char* server_address = nullptr; gpr_join_host_port(&server_address, "::", port_num); - builder.AddListeningPort(server_address, - Server::CreateServerCredentials(config)); + builder->AddListeningPort(server_address, + Server::CreateServerCredentials(config)); gpr_free(server_address); } - ApplyConfigToBuilder(config, &builder); + ApplyConfigToBuilder(config, builder.get()); - builder.RegisterService(&service_); + builder->RegisterService(&service_); - impl_ = builder.BuildAndStart(); + impl_ = builder->BuildAndStart(); } std::shared_ptr InProcessChannel( diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index bf3ddb42c92..072402b2cf6 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -7761,6 +7761,7 @@ "test/cpp/qps/histogram.h", "test/cpp/qps/interarrival.h", "test/cpp/qps/parse_json.h", + "test/cpp/qps/qps_server_builder.h", "test/cpp/qps/qps_worker.h", "test/cpp/qps/report.h", "test/cpp/qps/server.h", @@ -7782,6 +7783,8 @@ "test/cpp/qps/interarrival.h", "test/cpp/qps/parse_json.cc", "test/cpp/qps/parse_json.h", + "test/cpp/qps/qps_server_builder.cc", + "test/cpp/qps/qps_server_builder.h", "test/cpp/qps/qps_worker.cc", "test/cpp/qps/qps_worker.h", "test/cpp/qps/report.cc", From cebab02bf4ee3f8d38609d4c76030607073ca179 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 23 Jul 2018 19:12:29 +0200 Subject: [PATCH 35/61] C# native libraries should support Android 4.4+ (=api level 19) --- src/csharp/experimental/build_native_ext_for_android.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/csharp/experimental/build_native_ext_for_android.sh b/src/csharp/experimental/build_native_ext_for_android.sh index 5687a43a4e2..6518e03c53e 100755 --- a/src/csharp/experimental/build_native_ext_for_android.sh +++ b/src/csharp/experimental/build_native_ext_for_android.sh @@ -36,6 +36,7 @@ cd build # ANDROID_ABI in ('arm64-v8a', 'armeabi-v7a') # e.g. ANDROID_ABI=armeabi-v7a +# android-19 corresponds to Kitkat 4.4 ${ANDROID_SDK_CMAKE} ../.. \ -DCMAKE_TOOLCHAIN_FILE="${ANDROID_NDK_PATH}/build/cmake/android.toolchain.cmake" \ -DCMAKE_ANDROID_NDK="${ANDROID_NDK_PATH}" \ @@ -44,7 +45,7 @@ ${ANDROID_SDK_CMAKE} ../.. \ -DRUN_HAVE_STD_REGEX=0 \ -DRUN_HAVE_STEADY_CLOCK=0 \ -DCMAKE_BUILD_TYPE=Release \ - -DANDROID_PLATFORM=android-28 \ + -DANDROID_PLATFORM=android-19 \ -DANDROID_ABI="${ANDROID_ABI}" \ -DANDROID_NDK="${ANDROID_NDK_PATH}" From 0b907fc9d1cc37a69ab7877567154be8ab757e7f Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 23 Jul 2018 19:13:41 +0200 Subject: [PATCH 36/61] improve xamarin platform detection --- src/csharp/Grpc.Core/Internal/PlatformApis.cs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/csharp/Grpc.Core/Internal/PlatformApis.cs b/src/csharp/Grpc.Core/Internal/PlatformApis.cs index 6c4ee0bdb79..c501aa89fb6 100644 --- a/src/csharp/Grpc.Core/Internal/PlatformApis.cs +++ b/src/csharp/Grpc.Core/Internal/PlatformApis.cs @@ -33,8 +33,9 @@ namespace Grpc.Core.Internal internal static class PlatformApis { const string UnityEngineApplicationClassName = "UnityEngine.Application, UnityEngine"; - const string XamarinAndroidActivityClassName = "Android.App.Activity, Mono.Android"; - const string XamariniOSEnumClassName = "Mono.CSharp.Enum, Mono.CSharp"; + const string XamarinAndroidObjectClassName = "Java.Lang.Object, Mono.Android"; + const string XamarinIOSObjectClassName = "Foundation.NSObject, Xamarin.iOS"; + static readonly bool isLinux; static readonly bool isMacOSX; static readonly bool isWindows; @@ -42,7 +43,7 @@ namespace Grpc.Core.Internal static readonly bool isNetCore; static readonly bool isUnity; static readonly bool isXamarin; - static readonly bool isXamariniOS; + static readonly bool isXamarinIOS; static readonly bool isXamarinAndroid; static PlatformApis() @@ -63,9 +64,9 @@ namespace Grpc.Core.Internal #endif isMono = Type.GetType("Mono.Runtime") != null; isUnity = Type.GetType(UnityEngineApplicationClassName) != null; - isXamariniOS = Type.GetType(XamariniOSEnumClassName) != null; - isXamarinAndroid = Type.GetType(XamarinAndroidActivityClassName) != null; - isXamarin = isXamariniOS || isXamarinAndroid; + isXamarinIOS = Type.GetType(XamarinIOSObjectClassName) != null; + isXamarinAndroid = Type.GetType(XamarinAndroidObjectClassName) != null; + isXamarin = isXamarinIOS || isXamarinAndroid; } public static bool IsLinux @@ -108,9 +109,9 @@ namespace Grpc.Core.Internal /// /// true if running on Xamarin.iOS, false otherwise. /// - public static bool IsXamariniOS + public static bool IsXamarinIOS { - get { return isXamariniOS; } + get { return isXamarinIOS; } } /// From efe02c0dfb575ec4c0e13fd152c01c70d45df995 Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Mon, 23 Jul 2018 10:34:37 -0700 Subject: [PATCH 37/61] Generate directory index HTML file --- .../linux/grpc_publish_packages.sh | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tools/internal_ci/linux/grpc_publish_packages.sh b/tools/internal_ci/linux/grpc_publish_packages.sh index bf0c313599b..fecb9a5e09b 100644 --- a/tools/internal_ci/linux/grpc_publish_packages.sh +++ b/tools/internal_ci/linux/grpc_publish_packages.sh @@ -176,7 +176,55 @@ EOF tail --lines=+5 "$OLD_INDEX" }> "$NEW_INDEX" + +function generate_directory_index() +{ + local target_dir=$1 + local current_directory_name + current_directory_name=$(basename "$target_dir") + cat < + + + + Index of $current_directory_name - packages.grpc.io + + + +

Index of $current_directory_name

+
    +
  • .
    • +
  • ..
    • +EOF + +( + cd "$target_dir" + find * -maxdepth 0 -type d -print | sort | while read -r line + do + echo "
  • $line/
    • " + done + find * -maxdepth 0 -type f -print | sort | while read -r line + do + echo "
  • $line
    • " + done +) + +cat < + + +EOF +} + # Upload the current build artifacts gsutil -m cp -r "$LOCAL_STAGING_TEMPDIR/${BUILD_RELPATH%%/*}" "$GCS_ARCHIVE_ROOT" +# Upload directory indicies for subdirectories +( + cd "$LOCAL_BUILD_ROOT" + find * -type d | while read -r directory + do + generate_directory_index "$directory" | gsutil -h 'Content-Type:text/html' cp - "$GCS_ARCHIVE_ROOT$BUILD_RELPATH$directory/$INDEX_FILENAME" + done +) # Upload the new /index.xml gsutil -h "Content-Type:application/xml" cp "$NEW_INDEX" "$GCS_INDEX" From 081c27baad81f3f4862b5c89f0bf1bd792d19876 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 23 Jul 2018 22:59:02 +0200 Subject: [PATCH 38/61] add static_csharp rule to Makefile --- Makefile | 2 ++ templates/Makefile.template | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/Makefile b/Makefile index 2b4ed146ccc..bad41975a05 100644 --- a/Makefile +++ b/Makefile @@ -1381,6 +1381,8 @@ static_c: pc_c pc_c_unsecure cache.mk $(LIBDIR)/$(CONFIG)/libaddress_sorting.a static_cxx: pc_cxx pc_cxx_unsecure cache.mk $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc++_cronet.a $(LIBDIR)/$(CONFIG)/libgrpc++_error_details.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpcpp_channelz.a +static_csharp: static_c $(LIBDIR)/$(CONFIG)/libgrpc_csharp_ext.a + shared: shared_c shared_cxx shared_c: pc_c pc_c_unsecure cache.mk $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)address_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc_cronet$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) diff --git a/templates/Makefile.template b/templates/Makefile.template index 628056ed4db..50b81e5f9f2 100644 --- a/templates/Makefile.template +++ b/templates/Makefile.template @@ -921,6 +921,16 @@ % endfor + static_csharp: static_c \ + % for lib in libs: + % if 'Makefile' in lib.get('build_system', ['Makefile']): + % if lib.build == 'all' and lib.language == 'csharp': + $(LIBDIR)/$(CONFIG)/lib${lib.name}.a\ + % endif + % endif + % endfor + + shared: shared_c shared_cxx shared_c: pc_c pc_c_unsecure cache.mk\ From bcf271c416affe9871a43c8583d4295e8d73159a Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Mon, 23 Jul 2018 15:16:32 -0700 Subject: [PATCH 39/61] Allow error strings in final_info to propagate to filters on call destruction --- src/core/lib/channel/channel_stack.h | 2 +- src/core/lib/surface/call.cc | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/core/lib/channel/channel_stack.h b/src/core/lib/channel/channel_stack.h index 7581f937b64..35c3fb01eaa 100644 --- a/src/core/lib/channel/channel_stack.h +++ b/src/core/lib/channel/channel_stack.h @@ -82,7 +82,7 @@ typedef struct { typedef struct { grpc_call_stats stats; grpc_status_code final_status; - const char** error_string; + const char* error_string; } grpc_call_final_info; /* Channel filters specify: diff --git a/src/core/lib/surface/call.cc b/src/core/lib/surface/call.cc index 88e015ce22f..847bb87f7e5 100644 --- a/src/core/lib/surface/call.cc +++ b/src/core/lib/surface/call.cc @@ -561,7 +561,7 @@ static void destroy_call(void* call, grpc_error* error) { } get_final_status(c, set_status_value_directly, &c->final_info.final_status, - nullptr, c->final_info.error_string); + nullptr, &(c->final_info.error_string)); c->final_info.stats.latency = gpr_time_sub(gpr_now(GPR_CLOCK_MONOTONIC), c->start_time); @@ -573,6 +573,7 @@ static void destroy_call(void* call, grpc_error* error) { grpc_call_stack_destroy(CALL_STACK_FROM_CALL(c), &c->final_info, GRPC_CLOSURE_INIT(&c->release_call, release_call, c, grpc_schedule_on_exec_ctx)); + gpr_free(static_cast(const_cast(c->final_info.error_string))); } void grpc_call_ref(grpc_call* c) { gpr_ref(&c->ext_ref); } From 4ec08f723519123404f62089cfa1b02da1be4d89 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Mon, 23 Jul 2018 17:01:00 -0700 Subject: [PATCH 40/61] Add note on retrying RPCs Also add keepalive to list. --- doc/statuscodes.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/statuscodes.md b/doc/statuscodes.md index f2df9e00dec..9fbd58b1321 100644 --- a/doc/statuscodes.md +++ b/doc/statuscodes.md @@ -38,6 +38,7 @@ situations in which they are generated. | Error parsing response proto | INTERNAL | Client| | Error parsing request proto | INTERNAL | Server| | Sent or received message was larger than configured limit | RESOURCE_EXHAUSTED | Both | +| Keepalive watchdog times out | INTERNAL | Both | The following status codes are never generated by the library: - INVALID_ARGUMENT @@ -47,3 +48,5 @@ The following status codes are never generated by the library: - ABORTED - OUT_OF_RANGE - DATA_LOSS + +The decision to retry RPCs at the application level depends on the application and the type of error. There is no single guidance that will work for all. From e971fa4490bb98e9e35dac7b094204ddbd8a3f80 Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Mon, 23 Jul 2018 18:29:28 -0700 Subject: [PATCH 41/61] Add link to packages.grpc.io --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 1af9ec34f0a..1960f44137c 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,8 @@ For instructions on how to use the language-specific gRPC runtime in your projec You can find per-language quickstart guides and tutorials in [Documentation section on grpc.io website](https://grpc.io/docs/). The code examples are available in the [examples](examples) directory. +Precompiled bleeding-edge package builds of gRPC `master` branch's `HEAD` are uploaded daily to [packages.grpc.io](https://packages.grpc.io). + # To start developing gRPC Contributions are welcome! From 95390f943e8fb944f5f434b58cc3032b59f94eb8 Mon Sep 17 00:00:00 2001 From: Hope Casey-Allen Date: Thu, 19 Jul 2018 14:18:17 -0700 Subject: [PATCH 42/61] Replace atomics in gpr_arena_aloc with a mutex in the slow path --- src/core/lib/gpr/arena.cc | 88 ++++++++++++++++++++++++--------------- 1 file changed, 55 insertions(+), 33 deletions(-) diff --git a/src/core/lib/gpr/arena.cc b/src/core/lib/gpr/arena.cc index 141de69426c..e30b297aeab 100644 --- a/src/core/lib/gpr/arena.cc +++ b/src/core/lib/gpr/arena.cc @@ -25,6 +25,7 @@ #include #include #include +#include #include "src/core/lib/gpr/alloc.h" @@ -36,8 +37,6 @@ #ifdef SIMPLE_ARENA_FOR_DEBUGGING -#include - struct gpr_arena { gpr_mu mu; void** ptrs; @@ -78,14 +77,17 @@ void* gpr_arena_alloc(gpr_arena* arena, size_t size) { // would allow us to use the alignment actually needed by the caller. typedef struct zone { - size_t size_begin; - size_t size_end; - gpr_atm next_atm; + size_t size_begin; // All the space we have set aside for allocations up + // until this zone. + size_t size_end; // size_end = size_begin plus all the space we set aside for + // allocations in zone z itself. + zone* next; } zone; struct gpr_arena { gpr_atm size_so_far; zone initial_zone; + gpr_mu arena_growth_mutex; }; static void* zalloc_aligned(size_t size) { @@ -99,15 +101,17 @@ gpr_arena* gpr_arena_create(size_t initial_size) { gpr_arena* a = static_cast(zalloc_aligned( GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(gpr_arena)) + initial_size)); a->initial_zone.size_end = initial_size; + gpr_mu_init(&a->arena_growth_mutex); return a; } size_t gpr_arena_destroy(gpr_arena* arena) { + gpr_mu_destroy(&arena->arena_growth_mutex); gpr_atm size = gpr_atm_no_barrier_load(&arena->size_so_far); - zone* z = (zone*)gpr_atm_no_barrier_load(&arena->initial_zone.next_atm); + zone* z = arena->initial_zone.next; gpr_free_aligned(arena); while (z) { - zone* next_z = (zone*)gpr_atm_no_barrier_load(&z->next_atm); + zone* next_z = z->next; gpr_free_aligned(z); z = next_z; } @@ -116,37 +120,55 @@ size_t gpr_arena_destroy(gpr_arena* arena) { void* gpr_arena_alloc(gpr_arena* arena, size_t size) { size = GPR_ROUND_UP_TO_ALIGNMENT_SIZE(size); - size_t start = static_cast( + size_t previous_size_of_arena_allocations = static_cast( gpr_atm_no_barrier_fetch_add(&arena->size_so_far, size)); + size_t updated_size_of_arena_allocations = + previous_size_of_arena_allocations + size; zone* z = &arena->initial_zone; - while (start > z->size_end) { - zone* next_z = (zone*)gpr_atm_acq_load(&z->next_atm); - if (next_z == nullptr) { - size_t next_z_size = - static_cast(gpr_atm_no_barrier_load(&arena->size_so_far)); - next_z = static_cast(zalloc_aligned( - GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(zone)) + next_z_size)); - next_z->size_begin = z->size_end; - next_z->size_end = z->size_end + next_z_size; - if (!gpr_atm_rel_cas(&z->next_atm, static_cast(NULL), - (gpr_atm)next_z)) { - gpr_free_aligned(next_z); - next_z = (zone*)gpr_atm_acq_load(&z->next_atm); + // Check to see if the allocation isn't able to end in the initial zone. + // This statement is true only in the uncommon case because of our arena + // sizing historesis (that is, most calls should have a large enough initial + // zone and will not need to grow the arena). + if (updated_size_of_arena_allocations > z->size_end) { + // Find a zone to fit this allocation + gpr_mu_lock(&arena->arena_growth_mutex); + while (updated_size_of_arena_allocations > z->size_end) { + if (z->next == nullptr) { + // Note that we do an extra increment of size_so_far to prevent multiple + // simultaneous callers from stepping on each other. However, this extra + // increment means some space in the arena is wasted. + // So whenever we need to allocate x bytes and there are x - n (where + // n > 0) remaining in the current zone, we will waste x bytes (x - n + // in the current zone and n in the new zone). + previous_size_of_arena_allocations = static_cast( + gpr_atm_no_barrier_fetch_add(&arena->size_so_far, size)); + updated_size_of_arena_allocations = + previous_size_of_arena_allocations + size; + size_t next_z_size = updated_size_of_arena_allocations; + z->next = static_cast(zalloc_aligned( + GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(zone)) + next_z_size)); + z->next->size_begin = z->size_end; + z->next->size_end = z->size_end + next_z_size; } + z = z->next; } - z = next_z; - } - if (start + size > z->size_end) { - return gpr_arena_alloc(arena, size); + gpr_mu_unlock(&arena->arena_growth_mutex); } - GPR_ASSERT(start >= z->size_begin); - GPR_ASSERT(start + size <= z->size_end); - char* ptr = (z == &arena->initial_zone) - ? reinterpret_cast(arena) + - GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(gpr_arena)) - : reinterpret_cast(z) + - GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(zone)); - return ptr + start - z->size_begin; + GPR_ASSERT(previous_size_of_arena_allocations >= z->size_begin); + GPR_ASSERT(updated_size_of_arena_allocations <= z->size_end); + // Skip the first part of the zone, which just contains tracking information. + // For the initial zone, this is the gpr_arena struct and for any other zone, + // it's the zone struct. + char* start_of_allocation_space = + (z == &arena->initial_zone) + ? reinterpret_cast(arena) + + GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(gpr_arena)) + : reinterpret_cast(z) + + GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(zone)); + // previous_size_of_arena_allocations - size_begin is how many bytes have been + // allocated into the current zone + return start_of_allocation_space + previous_size_of_arena_allocations - + z->size_begin; } #endif // SIMPLE_ARENA_FOR_DEBUGGING From cbc2b1f40562b772c61e1448d4fa290587fca256 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 23 Jul 2018 23:13:34 +0200 Subject: [PATCH 43/61] experiment: build grpc_csharp_ext for ios --- .../experimental/build_native_ext_for_ios.sh | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100755 src/csharp/experimental/build_native_ext_for_ios.sh diff --git a/src/csharp/experimental/build_native_ext_for_ios.sh b/src/csharp/experimental/build_native_ext_for_ios.sh new file mode 100755 index 00000000000..518390ea63b --- /dev/null +++ b/src/csharp/experimental/build_native_ext_for_ios.sh @@ -0,0 +1,45 @@ +#!/bin/sh +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Helper script to crosscompile grpc_csharp_ext native extension for Android. + +set -ex + +cd "$(dirname "$0")/../../.." + +# +SDK="iphoneos" +# +ARCH="arm64" + +PATH_AR="$(xcrun --sdk $SDK --find ar)" +PATH_CC="$(xcrun --sdk $SDK --find clang)" +PATH_CXX="$(xcrun --sdk $SDK --find clang++)" + +# TODO(jtattermusch): add -mios-version-min=6.0 and -Wl,ios_version_min=6.0 +CPPFLAGS="-O2 -Wframe-larger-than=16384 -arch $ARCH -isysroot $(xcrun --sdk $SDK --show-sdk-path) -DPB_NO_PACKED_STRUCTS=1" +LDFLAGS="-arch $ARCH -isysroot $(xcrun --sdk $SDK --show-sdk-path)" + +# TODO(jtattermusch): revisit the build arguments +make -j4 static_csharp \ + VALID_CONFIG_ios_$ARCH="1" \ + CC_ios_$ARCH="$PATH_CC" \ + CXX_ios_$ARCH="$PATH_CXX" \ + LD_ios_$ARCH="$PATH_CC" \ + LDXX_ios_$ARCH="$PATH_CXX" \ + CPPFLAGS_ios_$ARCH="$CPPFLAGS" \ + LDFLAGS_ios_$ARCH="$LDFLAGS" \ + DEFINES_ios_$ARCH="NDEBUG" \ + CONFIG="ios_$ARCH" From 5bb8a33eb26a347bc790620552649c957dcc17e5 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 23 Jul 2018 23:14:11 +0200 Subject: [PATCH 44/61] copying files --- src/csharp/Grpc.Core/Grpc.Core.csproj | 12 ++++++++++++ .../Grpc.Core/build/Xamarin.iOS/Grpc.Core.targets | 15 +++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 src/csharp/Grpc.Core/build/Xamarin.iOS/Grpc.Core.targets diff --git a/src/csharp/Grpc.Core/Grpc.Core.csproj b/src/csharp/Grpc.Core/Grpc.Core.csproj index 0da95d203c6..7b0ae46dc18 100755 --- a/src/csharp/Grpc.Core/Grpc.Core.csproj +++ b/src/csharp/Grpc.Core/Grpc.Core.csproj @@ -54,6 +54,14 @@ runtimes/monoandroid/arm64-v8a/libgrpc_csharp_ext.so true + + runtimes/ios/native/libgrpc_csharp_ext.a + true + + + runtimes/ios/native/libgrpc.a + true + build/net45/ true @@ -62,6 +70,10 @@ build/MonoAndroid/ true + + build/Xamarin.iOS/ + true + diff --git a/src/csharp/Grpc.Core/build/Xamarin.iOS/Grpc.Core.targets b/src/csharp/Grpc.Core/build/Xamarin.iOS/Grpc.Core.targets new file mode 100644 index 00000000000..658158f6ea9 --- /dev/null +++ b/src/csharp/Grpc.Core/build/Xamarin.iOS/Grpc.Core.targets @@ -0,0 +1,15 @@ + + + + + + Static + True + + + Static + True + + + + From d9ad1c3f93540572b2336fdf584eed8bc87495b6 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 23 Jul 2018 23:26:56 +0200 Subject: [PATCH 45/61] building ios artifact --- tools/run_tests/artifacts/artifact_targets.py | 6 +++++ .../artifacts/build_artifact_csharp_ios.sh | 23 +++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100755 tools/run_tests/artifacts/build_artifact_csharp_ios.sh diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index 4500b220e97..3a1142dfee6 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -234,6 +234,11 @@ class CSharpExtArtifact: environ={ 'ANDROID_ABI': self.arch_abi }) + elif self.arch == 'ios': + return create_jobspec( + self.name, + ['tools/run_tests/artifacts/build_artifact_csharp_ios.sh'], + use_workspace=True) elif self.platform == 'windows': cmake_arch_option = 'Win32' if self.arch == 'x86' else self.arch return create_jobspec( @@ -356,6 +361,7 @@ def targets(): ] + [ CSharpExtArtifact('linux', 'android', arch_abi='arm64-v8a'), CSharpExtArtifact('linux', 'android', arch_abi='armeabi-v7a'), + CSharpExtArtifact('macos', 'ios'), PythonArtifact('linux', 'x86', 'cp27-cp27m'), PythonArtifact('linux', 'x86', 'cp27-cp27mu'), PythonArtifact('linux', 'x86', 'cp34-cp34m'), diff --git a/tools/run_tests/artifacts/build_artifact_csharp_ios.sh b/tools/run_tests/artifacts/build_artifact_csharp_ios.sh new file mode 100755 index 00000000000..8b3c7a17b38 --- /dev/null +++ b/tools/run_tests/artifacts/build_artifact_csharp_ios.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Copyright 2016 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +cd "$(dirname "$0")/../../.." + +src/csharp/experimental/build_native_ext_for_ios.sh + +mkdir -p "${ARTIFACTS_OUT}" +cp libs/ios_arm64/libgrpc_csharp_ext.a libs/ios_arm64/libgrpc.a "${ARTIFACTS_OUT}" From 756006f61113b0257d4f7a71f3ea994fbc53b1a3 Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Tue, 24 Jul 2018 01:11:53 -0700 Subject: [PATCH 46/61] Duplicate grpc_build_artifacts_extra job --- .../grpc_build_artifacts_extra_release.cfg | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 tools/internal_ci/linux/grpc_build_artifacts_extra_release.cfg diff --git a/tools/internal_ci/linux/grpc_build_artifacts_extra_release.cfg b/tools/internal_ci/linux/grpc_build_artifacts_extra_release.cfg new file mode 100644 index 00000000000..619e3ea3a97 --- /dev/null +++ b/tools/internal_ci/linux/grpc_build_artifacts_extra_release.cfg @@ -0,0 +1,26 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_build_artifacts_extra.sh" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} From 527d87add53b0ac7f13799facf0115c38846a23d Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 24 Jul 2018 11:36:25 +0200 Subject: [PATCH 47/61] fix Grpc.Core.csproj --- src/csharp/Grpc.Core/Grpc.Core.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/csharp/Grpc.Core/Grpc.Core.csproj b/src/csharp/Grpc.Core/Grpc.Core.csproj index 7b0ae46dc18..c4db3a3a875 100755 --- a/src/csharp/Grpc.Core/Grpc.Core.csproj +++ b/src/csharp/Grpc.Core/Grpc.Core.csproj @@ -54,11 +54,11 @@ runtimes/monoandroid/arm64-v8a/libgrpc_csharp_ext.so true - + runtimes/ios/native/libgrpc_csharp_ext.a true - + runtimes/ios/native/libgrpc.a true From 5aa2a0e376b22f35babf095ab1bc83078a7bf51b Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 24 Jul 2018 14:47:35 +0200 Subject: [PATCH 48/61] build fat binary for ios --- .../experimental/build_native_ext_for_ios.sh | 65 ++++++++++++------- .../artifacts/build_artifact_csharp_ios.sh | 2 +- 2 files changed, 42 insertions(+), 25 deletions(-) diff --git a/src/csharp/experimental/build_native_ext_for_ios.sh b/src/csharp/experimental/build_native_ext_for_ios.sh index 518390ea63b..69c9cdf021c 100755 --- a/src/csharp/experimental/build_native_ext_for_ios.sh +++ b/src/csharp/experimental/build_native_ext_for_ios.sh @@ -19,27 +19,44 @@ set -ex cd "$(dirname "$0")/../../.." -# -SDK="iphoneos" -# -ARCH="arm64" - -PATH_AR="$(xcrun --sdk $SDK --find ar)" -PATH_CC="$(xcrun --sdk $SDK --find clang)" -PATH_CXX="$(xcrun --sdk $SDK --find clang++)" - -# TODO(jtattermusch): add -mios-version-min=6.0 and -Wl,ios_version_min=6.0 -CPPFLAGS="-O2 -Wframe-larger-than=16384 -arch $ARCH -isysroot $(xcrun --sdk $SDK --show-sdk-path) -DPB_NO_PACKED_STRUCTS=1" -LDFLAGS="-arch $ARCH -isysroot $(xcrun --sdk $SDK --show-sdk-path)" - -# TODO(jtattermusch): revisit the build arguments -make -j4 static_csharp \ - VALID_CONFIG_ios_$ARCH="1" \ - CC_ios_$ARCH="$PATH_CC" \ - CXX_ios_$ARCH="$PATH_CXX" \ - LD_ios_$ARCH="$PATH_CC" \ - LDXX_ios_$ARCH="$PATH_CXX" \ - CPPFLAGS_ios_$ARCH="$CPPFLAGS" \ - LDFLAGS_ios_$ARCH="$LDFLAGS" \ - DEFINES_ios_$ARCH="NDEBUG" \ - CONFIG="ios_$ARCH" +# Usage: build +function build { + SDK="$1" + ARCH="$2" + + PATH_AR="$(xcrun --sdk $SDK --find ar)" + PATH_CC="$(xcrun --sdk $SDK --find clang)" + PATH_CXX="$(xcrun --sdk $SDK --find clang++)" + + # TODO(jtattermusch): add -mios-version-min=6.0 and -Wl,ios_version_min=6.0 + CPPFLAGS="-O2 -Wframe-larger-than=16384 -arch $ARCH -isysroot $(xcrun --sdk $SDK --show-sdk-path) -DPB_NO_PACKED_STRUCTS=1" + LDFLAGS="-arch $ARCH -isysroot $(xcrun --sdk $SDK --show-sdk-path)" + + # TODO(jtattermusch): revisit the build arguments + make -j4 static_csharp \ + VALID_CONFIG_ios_$ARCH="1" \ + CC_ios_$ARCH="$PATH_CC" \ + CXX_ios_$ARCH="$PATH_CXX" \ + LD_ios_$ARCH="$PATH_CC" \ + LDXX_ios_$ARCH="$PATH_CXX" \ + CPPFLAGS_ios_$ARCH="$CPPFLAGS" \ + LDFLAGS_ios_$ARCH="$LDFLAGS" \ + DEFINES_ios_$ARCH="NDEBUG" \ + CONFIG="ios_$ARCH" +} + +# Usage: fatten +function fatten { + LIB_NAME="$1" + + mkdir -p libs/ios + lipo -create -output libs/ios/lib$LIB_NAME.a \ + libs/ios_arm64/lib$LIB_NAME.a \ + libs/ios_x86_64/lib$LIB_NAME.a +} + +build iphoneos arm64 +build iphonesimulator x86_64 + +fatten grpc +fatten grpc_csharp_ext diff --git a/tools/run_tests/artifacts/build_artifact_csharp_ios.sh b/tools/run_tests/artifacts/build_artifact_csharp_ios.sh index 8b3c7a17b38..0d0a9c86da7 100755 --- a/tools/run_tests/artifacts/build_artifact_csharp_ios.sh +++ b/tools/run_tests/artifacts/build_artifact_csharp_ios.sh @@ -20,4 +20,4 @@ cd "$(dirname "$0")/../../.." src/csharp/experimental/build_native_ext_for_ios.sh mkdir -p "${ARTIFACTS_OUT}" -cp libs/ios_arm64/libgrpc_csharp_ext.a libs/ios_arm64/libgrpc.a "${ARTIFACTS_OUT}" +cp libs/ios/libgrpc_csharp_ext.a libs/ios/libgrpc.a "${ARTIFACTS_OUT}" From 20f36144be0b475e85e5772f87f51d42cd355d36 Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Tue, 24 Jul 2018 08:49:10 -0700 Subject: [PATCH 49/61] Add packages.grpc.io/web-assets to repo --- tools/package_hosting/404.html | 1 + tools/package_hosting/build-201807.xsl | 114 +++++++++++++++++++++++++ tools/package_hosting/dirindex.css | 16 ++++ tools/package_hosting/home.xsl | 86 +++++++++++++++++++ tools/package_hosting/style.css | 76 +++++++++++++++++ 5 files changed, 293 insertions(+) create mode 100644 tools/package_hosting/404.html create mode 100644 tools/package_hosting/build-201807.xsl create mode 100644 tools/package_hosting/dirindex.css create mode 100644 tools/package_hosting/home.xsl create mode 100644 tools/package_hosting/style.css diff --git a/tools/package_hosting/404.html b/tools/package_hosting/404.html new file mode 100644 index 00000000000..44d986c4b06 --- /dev/null +++ b/tools/package_hosting/404.html @@ -0,0 +1 @@ +404 Not Found diff --git a/tools/package_hosting/build-201807.xsl b/tools/package_hosting/build-201807.xsl new file mode 100644 index 00000000000..69a190446fa --- /dev/null +++ b/tools/package_hosting/build-201807.xsl @@ -0,0 +1,114 @@ + + + + + + + Artifacts for gRPC Build <xsl:value-of select="@id"/> + + + + + + + + + + + + + + + +
    + Artifacts for gRPC Build +
    +
    +
    + Build: + [invocation]
    + Timestamp: +
    + Branch: + + +
    + Commit: + +
    +
    + +
    +
    + +

    gRPC is a Cloud Native Computing Foundation project. Privacy Policy.

    +

    + Copyright ©  The gRPC Authors

    +
    +
    +
    + + +     + + +

    gRPC protoc Plugins

    + + + + +
    + +

    C#

    + + + + +
    + +

    PHP

    + + + + +
    + +

    Python

    + + + + + +
    + +

    Ruby

    + + + + +
    + +     + + + + + + + + + + + + +     diff --git a/tools/package_hosting/dirindex.css b/tools/package_hosting/dirindex.css new file mode 100644 index 00000000000..6c4b04ea323 --- /dev/null +++ b/tools/package_hosting/dirindex.css @@ -0,0 +1,16 @@ +ul { + list-style-type: none; +} +a{ + text-decoration: none; +} +a:hover { + text-decoration: underline; +} +ul li a { + font-family: 'SF Mono', 'Menlo', 'Monaco', 'Consolas', 'Courier New', Courier, monospace +} +h1 { + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, 'Open Sans', 'Helvetica Neue', sans-serif +} + diff --git a/tools/package_hosting/home.xsl b/tools/package_hosting/home.xsl new file mode 100644 index 00000000000..3f79303b1b7 --- /dev/null +++ b/tools/package_hosting/home.xsl @@ -0,0 +1,86 @@ + + + + + + + gRPC Packages + + + + + + + + + + + + + + +
    + gRPC Packages +
    +
    + + +
    +
    +

    gRPC is a Cloud Native Computing Foundation project. Privacy Policy.

    +

    Copyright © 2018 The gRPC Authors

    +
    + + +     + + +

    Official gRPC Releases

    +

    Commits corresponding to official gRPC release points and release candidates are tagged on GitHub.

    +

    To maximize usability, gRPC supports the standard way of adding dependencies in your language of choice (if there is one). + In most languages, the gRPC runtime comes in form of a package available in your language's package manager.

    +

    For instructions on how to use the language-specific gRPC runtime in your project, please refer to the following:

    + +     + + +

    Daily Builds of master Branch

    +

    gRPC packages are built on a daily basis at the HEAD of the master branch and are archived here.

    +

    + The current document (view source) is an XML feed pointing to the packages as they get built and uploaded. + You can subscribe to this feed and fetch, deploy, and test the precompiled packages with your continuous integration infrastructure. +

    +

    For stable release packages, please consult the above section and the common package manager for your language.

    + + + + + + + + + +
    TimestampCommitBuild ID
    +     + + + + + + + + + +     diff --git a/tools/package_hosting/style.css b/tools/package_hosting/style.css new file mode 100644 index 00000000000..dbd26d50cf5 --- /dev/null +++ b/tools/package_hosting/style.css @@ -0,0 +1,76 @@ +html, body +{ + margin: 0; + font-family: sans-serif; +} + +a, a:visited, a:link, a:active { + color: #2da6b0; + text-decoration: none; +} + +a:hover { + color: #2da6b0; + text-decoration: underline; +} + +#topbar { + background-color: #2da6b0; + height: 60px; + margin:auto; +} + +#topbar .title { + position: relative; + top: 24px; + left: 24px; + color: white; + font-family: sans-serif; + font-weight: bold; +} + +#main { + max-width:1100px; + margin:auto; +} + +#main h2 { + text-align: left; +} + +#main table { + width:100%; + border-collapse: collapse; + font-size: small; + font-family: 'SF Mono', 'Menlo', 'Monaco', 'Courier New', Courier, monospace; +} +#main table tr td { + border: solid black 1px; + padding: 5px; +} + +#main table tr td.hash { + text-align: right; + border-left: none; + font-size: x-small; +} + +#main table tr td.name { + text-align: left; + border-right: none; +} + +p.description +{ + font-size: smaller; +} + +#metadata { + margin-top: 15px; + padding: 15px; + font-family: 'SF Mono', 'Menlo', 'Monaco', 'Courier New', Courier, monospace; +} + +#metadata span.fieldname { + font-family: sans-serif; +} \ No newline at end of file From 628c0b5bed15e3a71c63f6fd6bd442d3317c0dc3 Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Tue, 24 Jul 2018 08:53:42 -0700 Subject: [PATCH 50/61] Add upload script for web-assets --- tools/package_hosting/upload_web_assets.sh | 30 ++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100755 tools/package_hosting/upload_web_assets.sh diff --git a/tools/package_hosting/upload_web_assets.sh b/tools/package_hosting/upload_web_assets.sh new file mode 100755 index 00000000000..dcf258e38eb --- /dev/null +++ b/tools/package_hosting/upload_web_assets.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +cd "$(dirname "$0")" + +GCS_WEB_ASSETS=gs://packages.grpc.io/web-assets/ + +WEB_ASSETS=( + 404.html + build-201807.xsl + dirindex.css + home.xsl + style.css +) + +gsutil -m cp "${WEB_ASSETS[@]}" "$GCS_WEB_ASSETS" From 056d061b3138ed26afe3132ed0d3f7d900d4735b Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 24 Jul 2018 19:29:04 +0200 Subject: [PATCH 51/61] Fix license year --- tools/run_tests/artifacts/build_artifact_csharp_ios.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/artifacts/build_artifact_csharp_ios.sh b/tools/run_tests/artifacts/build_artifact_csharp_ios.sh index 0d0a9c86da7..c902a45db7d 100755 --- a/tools/run_tests/artifacts/build_artifact_csharp_ios.sh +++ b/tools/run_tests/artifacts/build_artifact_csharp_ios.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2016 The gRPC Authors +# Copyright 2018 The gRPC Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From bcecc177a82ff150cfe98b42778180e3cff4d21d Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Tue, 24 Jul 2018 10:54:10 -0700 Subject: [PATCH 52/61] Update statuscodes.md Change wording --- doc/statuscodes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/statuscodes.md b/doc/statuscodes.md index 9fbd58b1321..2cce38295e4 100644 --- a/doc/statuscodes.md +++ b/doc/statuscodes.md @@ -49,4 +49,4 @@ The following status codes are never generated by the library: - OUT_OF_RANGE - DATA_LOSS -The decision to retry RPCs at the application level depends on the application and the type of error. There is no single guidance that will work for all. +Applications that may wish to retry failed RPCs must decide which status codes on which to retry. As shown in the table above, the gRPC library can generate the same status code for different cases. Server applications can also return those same status codes. Therefore, there is no fixed list of status codes on which it is appropriate to retry in all applications. As a result, individual applications must make their own determination as to which status codes should cause an RPC to be retried. From 600272c826b48420084c2ff76dfb0d34324ec296 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Tue, 24 Jul 2018 11:15:10 -0700 Subject: [PATCH 53/61] Update statuscodes.md Add link to retry doc --- doc/statuscodes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/statuscodes.md b/doc/statuscodes.md index 2cce38295e4..06fbe5c8fe8 100644 --- a/doc/statuscodes.md +++ b/doc/statuscodes.md @@ -49,4 +49,4 @@ The following status codes are never generated by the library: - OUT_OF_RANGE - DATA_LOSS -Applications that may wish to retry failed RPCs must decide which status codes on which to retry. As shown in the table above, the gRPC library can generate the same status code for different cases. Server applications can also return those same status codes. Therefore, there is no fixed list of status codes on which it is appropriate to retry in all applications. As a result, individual applications must make their own determination as to which status codes should cause an RPC to be retried. +Applications that may wish to [retry](https://github.com/grpc/proposal/blob/master/A6-client-retries.md) failed RPCs must decide which status codes on which to retry. As shown in the table above, the gRPC library can generate the same status code for different cases. Server applications can also return those same status codes. Therefore, there is no fixed list of status codes on which it is appropriate to retry in all applications. As a result, individual applications must make their own determination as to which status codes should cause an RPC to be retried. From 04fda25e6e2b6ecfbfd2f8df4db051fd184d0f3d Mon Sep 17 00:00:00 2001 From: Adele Zhou Date: Tue, 24 Jul 2018 12:11:16 -0700 Subject: [PATCH 54/61] Use n1-highmem-2 for most tests. --- third_party/toolchains/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/toolchains/BUILD b/third_party/toolchains/BUILD index 29fc4fdaf3c..02cd87a7b9b 100644 --- a/third_party/toolchains/BUILD +++ b/third_party/toolchains/BUILD @@ -42,7 +42,7 @@ platform( } properties: { name: "gceMachineType" # Small machines for majority of tests. - value: "n1-standard-1" + value: "n1-highmem-2" } properties: { name: "gceMachineType_LARGE" # Large machines for a small set of resource-consuming tests such as combiner_tests under TSAN. From 7cb30b0c88de17b869b961b642285247271034cc Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 24 Jul 2018 19:48:32 +0200 Subject: [PATCH 55/61] Grpc.Core: add support for x86 android emulator --- src/csharp/Grpc.Core/Grpc.Core.csproj | 4 ++++ src/csharp/Grpc.Core/build/MonoAndroid/Grpc.Core.targets | 7 +++++++ tools/run_tests/artifacts/artifact_targets.py | 1 + 3 files changed, 12 insertions(+) diff --git a/src/csharp/Grpc.Core/Grpc.Core.csproj b/src/csharp/Grpc.Core/Grpc.Core.csproj index c4db3a3a875..fc32271063f 100755 --- a/src/csharp/Grpc.Core/Grpc.Core.csproj +++ b/src/csharp/Grpc.Core/Grpc.Core.csproj @@ -54,6 +54,10 @@ runtimes/monoandroid/arm64-v8a/libgrpc_csharp_ext.so true     + + runtimes/monoandroid/x86/libgrpc_csharp_ext.so + true + runtimes/ios/native/libgrpc_csharp_ext.a true diff --git a/src/csharp/Grpc.Core/build/MonoAndroid/Grpc.Core.targets b/src/csharp/Grpc.Core/build/MonoAndroid/Grpc.Core.targets index f764f4cae1d..d75e5a2f2f9 100644 --- a/src/csharp/Grpc.Core/build/MonoAndroid/Grpc.Core.targets +++ b/src/csharp/Grpc.Core/build/MonoAndroid/Grpc.Core.targets @@ -18,4 +18,11 @@     + + + Always + x86 + + + diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index 3a1142dfee6..01323b53266 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -361,6 +361,7 @@ def targets(): ] + [ CSharpExtArtifact('linux', 'android', arch_abi='arm64-v8a'), CSharpExtArtifact('linux', 'android', arch_abi='armeabi-v7a'), + CSharpExtArtifact('linux', 'android', arch_abi='x86'), CSharpExtArtifact('macos', 'ios'), PythonArtifact('linux', 'x86', 'cp27-cp27m'), PythonArtifact('linux', 'x86', 'cp27-cp27mu'), From fd499c2baf0239ebe89bfd1d00a9543d4070658f Mon Sep 17 00:00:00 2001 From: Misty De Meo Date: Tue, 24 Jul 2018 14:11:05 -0700 Subject: [PATCH 56/61] Ruby tooling: fix makefile strip dependency --- src/ruby/ext/grpc/extconf.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ruby/ext/grpc/extconf.rb b/src/ruby/ext/grpc/extconf.rb index 4760f33e389..df8acc1bc26 100644 --- a/src/ruby/ext/grpc/extconf.rb +++ b/src/ruby/ext/grpc/extconf.rb @@ -110,7 +110,7 @@ if grpc_config == 'opt' o.puts i end o.puts - o.puts 'strip:' + o.puts 'strip: $(DLLIB)' o.puts "\t$(ECHO) Stripping $(DLLIB)" o.puts "\t$(Q) #{strip_tool} $(DLLIB)" end From ef29e14e1356bd7f8620b03e93c3dee7d83efaad Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Tue, 24 Jul 2018 14:13:27 -0700 Subject: [PATCH 57/61] Use test credentials provider for QPS benchmarking reporter client --- test/cpp/qps/benchmark_config.cc | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/test/cpp/qps/benchmark_config.cc b/test/cpp/qps/benchmark_config.cc index a4fd9de8206..5fd0f000389 100644 --- a/test/cpp/qps/benchmark_config.cc +++ b/test/cpp/qps/benchmark_config.cc @@ -22,6 +22,8 @@ #include #include +#include "test/cpp/util/test_credentials_provider.h" + DEFINE_bool(enable_log_reporter, true, "Enable reporting of benchmark results through GprLog"); @@ -44,6 +46,10 @@ DEFINE_string(rpc_reporter_server_address, "", DEFINE_bool(enable_rpc_reporter, false, "Enable use of RPC reporter"); +DEFINE_string( + rpc_reporter_credential_type, grpc::testing::kInsecureCredentialsType, + "Credential type for communication to the QPS benchmark report server"); + // In some distros, gflags is in the namespace google, and in some others, // in gflags. This hack is enabling us to find both. namespace google {} @@ -65,11 +71,14 @@ static std::shared_ptr InitBenchmarkReporters() { new JsonReporter("JsonReporter", FLAGS_scenario_result_file))); } if (FLAGS_enable_rpc_reporter) { + ChannelArguments channel_args; + std::shared_ptr channel_creds = + testing::GetCredentialsProvider()->GetChannelCredentials( + FLAGS_rpc_reporter_credential_type, &channel_args); GPR_ASSERT(!FLAGS_rpc_reporter_server_address.empty()); composite_reporter->add(std::unique_ptr(new RpcReporter( - "RpcReporter", - grpc::CreateChannel(FLAGS_rpc_reporter_server_address, - grpc::InsecureChannelCredentials())))); + "RpcReporter", grpc::CreateChannel(FLAGS_rpc_reporter_server_address, + channel_creds)))); } return std::shared_ptr(composite_reporter); From 4dbe919ec29e101344344a2b5fda70746ba0b77f Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Tue, 24 Jul 2018 17:50:37 -0700 Subject: [PATCH 58/61] Add version number to protoc packages --- tools/internal_ci/linux/grpc_publish_packages.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) mode change 100644 => 100755 tools/internal_ci/linux/grpc_publish_packages.sh diff --git a/tools/internal_ci/linux/grpc_publish_packages.sh b/tools/internal_ci/linux/grpc_publish_packages.sh old mode 100644 new mode 100755 index fecb9a5e09b..1459364d57c --- a/tools/internal_ci/linux/grpc_publish_packages.sh +++ b/tools/internal_ci/linux/grpc_publish_packages.sh @@ -17,6 +17,10 @@ set -ex shopt -s nullglob +cd "$(dirname "$0")/../../.." + +GRPC_VERSION=$(grep -e "^ *version: " build.yaml | head -n 1 | sed 's/.*: //') + INPUT_ARTIFACTS=$KOKORO_GFILE_DIR/github/grpc/artifacts INDEX_FILENAME=index.xml @@ -43,17 +47,17 @@ find "$INPUT_ARTIFACTS" -type f PROTOC_PLUGINS_ZIPPED_PACKAGES=$(mktemp -d) for zip_dir in protoc_windows_{x86,x64} do - zip -jr "$PROTOC_PLUGINS_ZIPPED_PACKAGES/$zip_dir.zip" "$INPUT_ARTIFACTS/$zip_dir/"* + zip -jr "$PROTOC_PLUGINS_ZIPPED_PACKAGES/grpc-$zip_dir-$GRPC_VERSION.zip" "$INPUT_ARTIFACTS/$zip_dir/"* done for tar_dir in protoc_{linux,macos}_{x86,x64} do chmod +x "$INPUT_ARTIFACTS/$tar_dir"/* - tar -cvzf "$PROTOC_PLUGINS_ZIPPED_PACKAGES/$tar_dir.tar.gz" -C "$INPUT_ARTIFACTS/$tar_dir" . + tar -cvzf "$PROTOC_PLUGINS_ZIPPED_PACKAGES/grpc-$tar_dir-$GRPC_VERSION.tar.gz" -C "$INPUT_ARTIFACTS/$tar_dir" . done PROTOC_PACKAGES=( - "$PROTOC_PLUGINS_ZIPPED_PACKAGES"/protoc_windows_{x86,x64}.zip - "$PROTOC_PLUGINS_ZIPPED_PACKAGES"/protoc_{linux,macos}_{x86,x64}.tar.gz + "$PROTOC_PLUGINS_ZIPPED_PACKAGES"/grpc-protoc_windows_{x86,x64}-"$GRPC_VERSION.zip" + "$PROTOC_PLUGINS_ZIPPED_PACKAGES"/grpc-protoc_{linux,macos}_{x86,x64}-"$GRPC_VERSION.tar.gz" ) # C# From b0e41f6c7da8015935fdda3ce38a728f01be4802 Mon Sep 17 00:00:00 2001 From: Juanli Shen Date: Tue, 24 Jul 2018 20:54:48 -0700 Subject: [PATCH 59/61] Fix re-resolution in pick_first --- .../lb_policy/pick_first/pick_first.cc | 16 ++++---- .../ext/filters/client_channel/subchannel.cc | 14 +++---- test/cpp/end2end/client_lb_end2end_test.cc | 40 ++++++++++++++++++- 3 files changed, 53 insertions(+), 17 deletions(-) diff --git a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc index 023281db974..d217dc0e630 100644 --- a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc +++ b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc @@ -451,6 +451,7 @@ void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked( // latest pending subchannel lists. GPR_ASSERT(subchannel_list() == p->subchannel_list_.get() || subchannel_list() == p->latest_pending_subchannel_list_.get()); + GPR_ASSERT(connectivity_state != GRPC_CHANNEL_SHUTDOWN); // Handle updates for the currently selected subchannel. if (p->selected_ == this) { if (grpc_lb_pick_first_trace.enabled()) { @@ -480,14 +481,12 @@ void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked( "update"), "selected_not_ready+switch_to_update"); } else { - // TODO(juanlishen): we re-resolve when the selected subchannel goes to - // TRANSIENT_FAILURE because we used to shut down in this case before - // re-resolution is introduced. But we need to investigate whether we - // really want to take any action instead of waiting for the selected - // subchannel reconnecting. - GPR_ASSERT(connectivity_state != GRPC_CHANNEL_SHUTDOWN); if (connectivity_state == GRPC_CHANNEL_TRANSIENT_FAILURE) { - // If the selected channel goes bad, request a re-resolution. + // If the selected subchannel goes bad, request a re-resolution. We also + // set the channel state to IDLE and reset started_picking_. The reason + // is that if the new state is TRANSIENT_FAILURE due to a GOAWAY + // reception we don't want to connect to the re-resolved backends until + // we leave the IDLE state. grpc_connectivity_state_set(&p->state_tracker_, GRPC_CHANNEL_IDLE, GRPC_ERROR_NONE, "selected_changed+reresolve"); @@ -568,9 +567,10 @@ void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked( // Case 1: Only set state to TRANSIENT_FAILURE if we've tried // all subchannels. if (sd->Index() == 0 && subchannel_list() == p->subchannel_list_.get()) { + p->TryReresolutionLocked(&grpc_lb_pick_first_trace, GRPC_ERROR_NONE); grpc_connectivity_state_set( &p->state_tracker_, GRPC_CHANNEL_TRANSIENT_FAILURE, - GRPC_ERROR_REF(error), "connecting_transient_failure"); + GRPC_ERROR_REF(error), "exhausted_subchannels"); } sd->StartConnectivityWatchLocked(); break; diff --git a/src/core/ext/filters/client_channel/subchannel.cc b/src/core/ext/filters/client_channel/subchannel.cc index 93df2aff700..71ef8c518bb 100644 --- a/src/core/ext/filters/client_channel/subchannel.cc +++ b/src/core/ext/filters/client_channel/subchannel.cc @@ -402,8 +402,6 @@ static void continue_connect_locked(grpc_subchannel* c) { c->next_attempt_deadline = c->backoff->NextAttemptTime(); args.deadline = std::max(c->next_attempt_deadline, min_deadline); args.channel_args = c->args; - grpc_connectivity_state_set(&c->state_tracker, GRPC_CHANNEL_CONNECTING, - GRPC_ERROR_NONE, "state_change"); grpc_connector_connect(c->connector, &args, &c->connecting_result, &c->on_connected); } @@ -459,27 +457,24 @@ static void maybe_start_connecting_locked(grpc_subchannel* c) { /* Don't try to connect if we're already disconnected */ return; } - if (c->connecting) { /* Already connecting: don't restart */ return; } - if (c->connected_subchannel != nullptr) { /* Already connected: don't restart */ return; } - if (!grpc_connectivity_state_has_watchers(&c->state_tracker)) { /* Nobody is interested in connecting: so don't just yet */ return; } - c->connecting = true; GRPC_SUBCHANNEL_WEAK_REF(c, "connecting"); - if (!c->backoff_begun) { c->backoff_begun = true; + grpc_connectivity_state_set(&c->state_tracker, GRPC_CHANNEL_CONNECTING, + GRPC_ERROR_NONE, "connecting"); continue_connect_locked(c); } else { GPR_ASSERT(!c->have_alarm); @@ -494,6 +489,11 @@ static void maybe_start_connecting_locked(grpc_subchannel* c) { } GRPC_CLOSURE_INIT(&c->on_alarm, on_alarm, c, grpc_schedule_on_exec_ctx); grpc_timer_init(&c->alarm, c->next_attempt_deadline, &c->on_alarm); + // During backoff, we prefer the connectivity state of CONNECTING instead of + // TRANSIENT_FAILURE in order to prevent triggering re-resolution + // continuously in pick_first. + grpc_connectivity_state_set(&c->state_tracker, GRPC_CHANNEL_CONNECTING, + GRPC_ERROR_NONE, "backoff"); } } diff --git a/test/cpp/end2end/client_lb_end2end_test.cc b/test/cpp/end2end/client_lb_end2end_test.cc index 8896fc6cae7..c5a73a24698 100644 --- a/test/cpp/end2end/client_lb_end2end_test.cc +++ b/test/cpp/end2end/client_lb_end2end_test.cc @@ -279,9 +279,14 @@ class ClientLbEnd2endTest : public ::testing::Test { void WaitForServer( const std::unique_ptr& stub, - size_t server_idx, const grpc_core::DebugLocation& location) { + size_t server_idx, const grpc_core::DebugLocation& location, + bool ignore_failure = false) { do { - CheckRpcSendOk(stub, location); + if (ignore_failure) { + SendRpc(stub); + } else { + CheckRpcSendOk(stub, location); + } } while (servers_[server_idx]->service_.request_count() == 0); ResetCounters(); } @@ -507,6 +512,37 @@ TEST_F(ClientLbEnd2endTest, PickFirstManyUpdates) { EXPECT_EQ("pick_first", channel->GetLoadBalancingPolicyName()); } +TEST_F(ClientLbEnd2endTest, PickFirstReresolutionNoSelected) { + // Prepare the ports for up servers and down servers. + const int kNumServers = 3; + const int kNumAliveServers = 1; + StartServers(kNumAliveServers); + std::vector alive_ports, dead_ports; + for (size_t i = 0; i < kNumServers; ++i) { + if (i < kNumAliveServers) { + alive_ports.emplace_back(servers_[i]->port_); + } else { + dead_ports.emplace_back(grpc_pick_unused_port_or_die()); + } + } + auto channel = BuildChannel("pick_first"); + auto stub = BuildStub(channel); + // The initial resolution only contains dead ports. There won't be any + // selected subchannel. Re-resolution will return the same result. + SetNextResolution(dead_ports); + gpr_log(GPR_INFO, "****** INITIAL RESOLUTION SET *******"); + for (size_t i = 0; i < 10; ++i) CheckRpcSendFailure(stub); + // Set a re-resolution result that contains reachable ports, so that the + // pick_first LB policy can recover soon. + SetNextResolutionUponError(alive_ports); + gpr_log(GPR_INFO, "****** RE-RESOLUTION SET *******"); + WaitForServer(stub, 0, DEBUG_LOCATION, true /* ignore_failure */); + CheckRpcSendOk(stub, DEBUG_LOCATION); + EXPECT_EQ(servers_[0]->service_.request_count(), 1); + // Check LB policy name for the channel. + EXPECT_EQ("pick_first", channel->GetLoadBalancingPolicyName()); +} + TEST_F(ClientLbEnd2endTest, RoundRobin) { // Start servers and send one RPC per server. const int kNumServers = 3; From a223a2c554c488a59745911c819e5e8353b4bb55 Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Wed, 25 Jul 2018 10:27:32 -0700 Subject: [PATCH 60/61] Include commit hash in directory name for archived packages --- tools/internal_ci/linux/grpc_publish_packages.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/internal_ci/linux/grpc_publish_packages.sh b/tools/internal_ci/linux/grpc_publish_packages.sh index 1459364d57c..ef943db7e15 100755 --- a/tools/internal_ci/linux/grpc_publish_packages.sh +++ b/tools/internal_ci/linux/grpc_publish_packages.sh @@ -28,7 +28,7 @@ BUILD_ID=${KOKORO_BUILD_ID:-$(uuidgen)} BUILD_BRANCH_NAME=master BUILD_GIT_COMMIT=${KOKORO_GIT_COMMIT:-unknown} BUILD_TIMESTAMP=$(date -Iseconds) -BUILD_RELPATH=$(date "+%Y/%m")/$BUILD_ID/ +BUILD_RELPATH=$(date "+%Y/%m")/$BUILD_GIT_COMMIT-$BUILD_ID/ GCS_ROOT=gs://packages.grpc.io/ GCS_ARCHIVE_PREFIX=archive/ From b928f99fdf53e35a1636f943910fc6a7315b8d0f Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Wed, 25 Jul 2018 10:47:17 -0700 Subject: [PATCH 61/61] Disable ChannelConnectivityTest for the Beta API The Beta API has been unsupported for a while and we plan to disable the flaky tests in the Beta API as we see them before we entirely remove it. --- src/python/grpcio_tests/tests/unit/beta/_utilities_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/python/grpcio_tests/tests/unit/beta/_utilities_test.py b/src/python/grpcio_tests/tests/unit/beta/_utilities_test.py index aebee4da965..e0422627962 100644 --- a/src/python/grpcio_tests/tests/unit/beta/_utilities_test.py +++ b/src/python/grpcio_tests/tests/unit/beta/_utilities_test.py @@ -41,6 +41,7 @@ class _Callback(object): return self._value +@unittest.skip('https://github.com/grpc/grpc/issues/16134') class ChannelConnectivityTest(unittest.TestCase): def test_lonely_channel_connectivity(self):