From 27b014ef351fdc87c66637793f07b88608b924af Mon Sep 17 00:00:00 2001 From: Srini Polavarapu Date: Thu, 19 Jul 2018 09:48:58 -0700 Subject: [PATCH 01/25] Bump version to 1.15.0-dev --- BUILD | 4 ++-- build.yaml | 4 ++-- doc/g_stands_for.md | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/BUILD b/BUILD index 8523bbb660f..32c01cd281e 100644 --- a/BUILD +++ b/BUILD @@ -64,11 +64,11 @@ config_setting( ) # This should be updated along with build.yaml -g_stands_for = "gladiolus" +g_stands_for = "glider" core_version = "6.0.0-dev" -version = "1.14.0-dev" +version = "1.15.0-dev" GPR_PUBLIC_HDRS = [ "include/grpc/support/alloc.h", diff --git a/build.yaml b/build.yaml index 3067ca9161d..afcbd4dcee2 100644 --- a/build.yaml +++ b/build.yaml @@ -13,8 +13,8 @@ settings: '#09': Per-language overrides are possible with (eg) ruby_version tag here '#10': See the expand_version.py for all the quirks here core_version: 6.0.0-dev - g_stands_for: gladiolus - version: 1.14.0-dev + g_stands_for: glider + version: 1.15.0-dev filegroups: - name: alts_proto headers: diff --git a/doc/g_stands_for.md b/doc/g_stands_for.md index f86bd76f328..b3ed9bd7138 100644 --- a/doc/g_stands_for.md +++ b/doc/g_stands_for.md @@ -13,4 +13,5 @@ - 1.11 'g' stands for ['gorgeous'](https://github.com/grpc/grpc/tree/v1.11.x) - 1.12 'g' stands for ['glorious'](https://github.com/grpc/grpc/tree/v1.12.x) - 1.13 'g' stands for ['gloriosa'](https://github.com/grpc/grpc/tree/v1.13.x) -- 1.14 'g' stands for ['gladiolus'](https://github.com/grpc/grpc/tree/master) +- 1.14 'g' stands for ['gladiolus'](https://github.com/grpc/grpc/tree/v1.14.x) +- 1.15 'g' stands for ['glider'](https://github.com/grpc/grpc/tree/master) From 8c9af7464c57d8a1d9cdb6505beb4baba179ee47 Mon Sep 17 00:00:00 2001 From: Srini Polavarapu Date: Thu, 19 Jul 2018 09:52:56 -0700 Subject: [PATCH 02/25] Regenerate projects --- CMakeLists.txt | 2 +- Makefile | 4 ++-- gRPC-C++.podspec | 4 ++-- gRPC-Core.podspec | 2 +- gRPC-ProtoRPC.podspec | 2 +- gRPC-RxLibrary.podspec | 2 +- gRPC.podspec | 2 +- package.xml | 4 ++-- src/core/lib/surface/version.cc | 2 +- src/cpp/common/version_cc.cc | 2 +- src/csharp/Grpc.Core/Version.csproj.include | 2 +- src/csharp/Grpc.Core/VersionInfo.cs | 4 ++-- src/csharp/build_packages_dotnetcli.bat | 2 +- src/csharp/build_packages_dotnetcli.sh | 6 +++--- src/objective-c/!ProtoCompiler-gRPCPlugin.podspec | 2 +- src/objective-c/GRPCClient/private/version.h | 2 +- src/objective-c/tests/version.h | 2 +- src/php/composer.json | 2 +- src/php/ext/grpc/version.h | 2 +- src/python/grpcio/grpc/_grpcio_metadata.py | 2 +- src/python/grpcio/grpc_version.py | 2 +- src/python/grpcio_health_checking/grpc_version.py | 2 +- src/python/grpcio_reflection/grpc_version.py | 2 +- src/python/grpcio_testing/grpc_version.py | 2 +- src/python/grpcio_tests/grpc_version.py | 2 +- src/ruby/lib/grpc/version.rb | 2 +- src/ruby/tools/version.rb | 2 +- tools/distrib/python/grpcio_tools/grpc_version.py | 2 +- tools/doxygen/Doxyfile.c++ | 2 +- tools/doxygen/Doxyfile.c++.internal | 2 +- 30 files changed, 36 insertions(+), 36 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7222894af62..e3df19be5b4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,7 +24,7 @@ cmake_minimum_required(VERSION 2.8) set(PACKAGE_NAME "grpc") -set(PACKAGE_VERSION "1.14.0-dev") +set(PACKAGE_VERSION "1.15.0-dev") set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}") set(PACKAGE_TARNAME "${PACKAGE_NAME}-${PACKAGE_VERSION}") set(PACKAGE_BUGREPORT "https://github.com/grpc/grpc/issues/") diff --git a/Makefile b/Makefile index 47c9dc7ccdf..8d0a9d91816 100644 --- a/Makefile +++ b/Makefile @@ -437,8 +437,8 @@ Q = @ endif CORE_VERSION = 6.0.0-dev -CPP_VERSION = 1.14.0-dev -CSHARP_VERSION = 1.14.0-dev +CPP_VERSION = 1.15.0-dev +CSHARP_VERSION = 1.15.0-dev CPPFLAGS_NO_ARCH += $(addprefix -I, $(INCLUDES)) $(addprefix -D, $(DEFINES)) CPPFLAGS += $(CPPFLAGS_NO_ARCH) $(ARCH_FLAGS) diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index 57d58cc4400..1d9237bf627 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -23,7 +23,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-C++' # TODO (mxyan): use version that match gRPC version when pod is stabilized - # version = '1.14.0-dev' + # version = '1.15.0-dev' version = '0.0.3' s.version = version s.summary = 'gRPC C++ library' @@ -31,7 +31,7 @@ Pod::Spec.new do |s| s.license = 'Apache License, Version 2.0' s.authors = { 'The gRPC contributors' => 'grpc-packages@google.com' } - grpc_version = '1.14.0-dev' + grpc_version = '1.15.0-dev' s.source = { :git => 'https://github.com/grpc/grpc.git', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 997617c5307..23edaec656d 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -22,7 +22,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-Core' - version = '1.14.0-dev' + version = '1.15.0-dev' s.version = version s.summary = 'Core cross-platform gRPC library, written in C' s.homepage = 'https://grpc.io' diff --git a/gRPC-ProtoRPC.podspec b/gRPC-ProtoRPC.podspec index 6548f36a32b..79ec679e3aa 100644 --- a/gRPC-ProtoRPC.podspec +++ b/gRPC-ProtoRPC.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-ProtoRPC' - version = '1.14.0-dev' + version = '1.15.0-dev' s.version = version s.summary = 'RPC library for Protocol Buffers, based on gRPC' s.homepage = 'https://grpc.io' diff --git a/gRPC-RxLibrary.podspec b/gRPC-RxLibrary.podspec index ebd942cf785..7a461e270f9 100644 --- a/gRPC-RxLibrary.podspec +++ b/gRPC-RxLibrary.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-RxLibrary' - version = '1.14.0-dev' + version = '1.15.0-dev' s.version = version s.summary = 'Reactive Extensions library for iOS/OSX.' s.homepage = 'https://grpc.io' diff --git a/gRPC.podspec b/gRPC.podspec index d6d59c21415..bf3f56bd549 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -20,7 +20,7 @@ Pod::Spec.new do |s| s.name = 'gRPC' - version = '1.14.0-dev' + version = '1.15.0-dev' s.version = version s.summary = 'gRPC client library for iOS/OSX' s.homepage = 'https://grpc.io' diff --git a/package.xml b/package.xml index a9dc2dc79ac..7f71536b1d0 100644 --- a/package.xml +++ b/package.xml @@ -13,8 +13,8 @@ 2018-01-19 - 1.14.0dev - 1.14.0dev + 1.15.0dev + 1.15.0dev beta diff --git a/src/core/lib/surface/version.cc b/src/core/lib/surface/version.cc index ac8cec29804..e92fe2c5a15 100644 --- a/src/core/lib/surface/version.cc +++ b/src/core/lib/surface/version.cc @@ -25,4 +25,4 @@ const char* grpc_version_string(void) { return "6.0.0-dev"; } -const char* grpc_g_stands_for(void) { return "gladiolus"; } +const char* grpc_g_stands_for(void) { return "glider"; } diff --git a/src/cpp/common/version_cc.cc b/src/cpp/common/version_cc.cc index a7b093dfd2a..b8fca9a6ee3 100644 --- a/src/cpp/common/version_cc.cc +++ b/src/cpp/common/version_cc.cc @@ -22,5 +22,5 @@ #include namespace grpc { -grpc::string Version() { return "1.14.0-dev"; } +grpc::string Version() { return "1.15.0-dev"; } } // namespace grpc diff --git a/src/csharp/Grpc.Core/Version.csproj.include b/src/csharp/Grpc.Core/Version.csproj.include index 2b45e7ae203..1a746acdd4f 100755 --- a/src/csharp/Grpc.Core/Version.csproj.include +++ b/src/csharp/Grpc.Core/Version.csproj.include @@ -1,7 +1,7 @@ - 1.14.0-dev + 1.15.0-dev 3.5.1 diff --git a/src/csharp/Grpc.Core/VersionInfo.cs b/src/csharp/Grpc.Core/VersionInfo.cs index c4531e5275a..295e0f2577b 100644 --- a/src/csharp/Grpc.Core/VersionInfo.cs +++ b/src/csharp/Grpc.Core/VersionInfo.cs @@ -33,11 +33,11 @@ namespace Grpc.Core /// /// Current AssemblyFileVersion of gRPC C# assemblies /// - public const string CurrentAssemblyFileVersion = "1.14.0.0"; + public const string CurrentAssemblyFileVersion = "1.15.0.0"; /// /// Current version of gRPC C# /// - public const string CurrentVersion = "1.14.0-dev"; + public const string CurrentVersion = "1.15.0-dev"; } } diff --git a/src/csharp/build_packages_dotnetcli.bat b/src/csharp/build_packages_dotnetcli.bat index 394b859e0be..67af258354f 100755 --- a/src/csharp/build_packages_dotnetcli.bat +++ b/src/csharp/build_packages_dotnetcli.bat @@ -13,7 +13,7 @@ @rem limitations under the License. @rem Current package versions -set VERSION=1.14.0-dev +set VERSION=1.15.0-dev @rem Adjust the location of nuget.exe set NUGET=C:\nuget\nuget.exe diff --git a/src/csharp/build_packages_dotnetcli.sh b/src/csharp/build_packages_dotnetcli.sh index 273d745f170..88b4d63538d 100755 --- a/src/csharp/build_packages_dotnetcli.sh +++ b/src/csharp/build_packages_dotnetcli.sh @@ -45,8 +45,8 @@ dotnet pack --configuration Release Grpc.Auth --output ../../../artifacts dotnet pack --configuration Release Grpc.HealthCheck --output ../../../artifacts dotnet pack --configuration Release Grpc.Reflection --output ../../../artifacts -nuget pack Grpc.nuspec -Version "1.14.0-dev" -OutputDirectory ../../artifacts -nuget pack Grpc.Core.NativeDebug.nuspec -Version "1.14.0-dev" -OutputDirectory ../../artifacts -nuget pack Grpc.Tools.nuspec -Version "1.14.0-dev" -OutputDirectory ../../artifacts +nuget pack Grpc.nuspec -Version "1.15.0-dev" -OutputDirectory ../../artifacts +nuget pack Grpc.Core.NativeDebug.nuspec -Version "1.15.0-dev" -OutputDirectory ../../artifacts +nuget pack Grpc.Tools.nuspec -Version "1.15.0-dev" -OutputDirectory ../../artifacts (cd ../../artifacts && zip csharp_nugets_dotnetcli.zip *.nupkg) diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index 28dfffd8631..6ad9166b32e 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -42,7 +42,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler-gRPCPlugin' - v = '1.14.0-dev' + v = '1.15.0-dev' s.version = v s.summary = 'The gRPC ProtoC plugin generates Objective-C files from .proto services.' s.description = <<-DESC diff --git a/src/objective-c/GRPCClient/private/version.h b/src/objective-c/GRPCClient/private/version.h index 60571b1f43f..52fe0dd0502 100644 --- a/src/objective-c/GRPCClient/private/version.h +++ b/src/objective-c/GRPCClient/private/version.h @@ -22,4 +22,4 @@ // instead. This file can be regenerated from the template by running // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.14.0-dev" +#define GRPC_OBJC_VERSION_STRING @"1.15.0-dev" diff --git a/src/objective-c/tests/version.h b/src/objective-c/tests/version.h index fafcac6507d..d532eed245d 100644 --- a/src/objective-c/tests/version.h +++ b/src/objective-c/tests/version.h @@ -22,5 +22,5 @@ // instead. This file can be regenerated from the template by running // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.14.0-dev" +#define GRPC_OBJC_VERSION_STRING @"1.15.0-dev" #define GRPC_C_VERSION_STRING @"6.0.0-dev" diff --git a/src/php/composer.json b/src/php/composer.json index 94918417d9f..b6716b8b4c3 100644 --- a/src/php/composer.json +++ b/src/php/composer.json @@ -2,7 +2,7 @@ "name": "grpc/grpc-dev", "description": "gRPC library for PHP - for Developement use only", "license": "Apache-2.0", - "version": "1.14.0", + "version": "1.15.0", "require": { "php": ">=5.5.0", "google/protobuf": "^v3.3.0" diff --git a/src/php/ext/grpc/version.h b/src/php/ext/grpc/version.h index c985863cb59..99dd9d68f05 100644 --- a/src/php/ext/grpc/version.h +++ b/src/php/ext/grpc/version.h @@ -20,6 +20,6 @@ #ifndef VERSION_H #define VERSION_H -#define PHP_GRPC_VERSION "1.14.0dev" +#define PHP_GRPC_VERSION "1.15.0dev" #endif /* VERSION_H */ diff --git a/src/python/grpcio/grpc/_grpcio_metadata.py b/src/python/grpcio/grpc/_grpcio_metadata.py index b336e6aae54..c33911ebc1b 100644 --- a/src/python/grpcio/grpc/_grpcio_metadata.py +++ b/src/python/grpcio/grpc/_grpcio_metadata.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc/_grpcio_metadata.py.template`!!! -__version__ = """1.14.0.dev0""" +__version__ = """1.15.0.dev0""" diff --git a/src/python/grpcio/grpc_version.py b/src/python/grpcio/grpc_version.py index 2eeaabc8b88..9337800a332 100644 --- a/src/python/grpcio/grpc_version.py +++ b/src/python/grpcio/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_version.py.template`!!! -VERSION = '1.14.0.dev0' +VERSION = '1.15.0.dev0' diff --git a/src/python/grpcio_health_checking/grpc_version.py b/src/python/grpcio_health_checking/grpc_version.py index f36de8d0fa1..3b84f7a4c55 100644 --- a/src/python/grpcio_health_checking/grpc_version.py +++ b/src/python/grpcio_health_checking/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_health_checking/grpc_version.py.template`!!! -VERSION = '1.14.0.dev0' +VERSION = '1.15.0.dev0' diff --git a/src/python/grpcio_reflection/grpc_version.py b/src/python/grpcio_reflection/grpc_version.py index 2249b07d9d7..7b0e48ea23b 100644 --- a/src/python/grpcio_reflection/grpc_version.py +++ b/src/python/grpcio_reflection/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_reflection/grpc_version.py.template`!!! -VERSION = '1.14.0.dev0' +VERSION = '1.15.0.dev0' diff --git a/src/python/grpcio_testing/grpc_version.py b/src/python/grpcio_testing/grpc_version.py index a5b275aff65..df9953fa257 100644 --- a/src/python/grpcio_testing/grpc_version.py +++ b/src/python/grpcio_testing/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_testing/grpc_version.py.template`!!! -VERSION = '1.14.0.dev0' +VERSION = '1.15.0.dev0' diff --git a/src/python/grpcio_tests/grpc_version.py b/src/python/grpcio_tests/grpc_version.py index 816dfb55bc1..b2cf129e4fb 100644 --- a/src/python/grpcio_tests/grpc_version.py +++ b/src/python/grpcio_tests/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_tests/grpc_version.py.template`!!! -VERSION = '1.14.0.dev0' +VERSION = '1.15.0.dev0' diff --git a/src/ruby/lib/grpc/version.rb b/src/ruby/lib/grpc/version.rb index 902d59bd71a..5c21a5b1685 100644 --- a/src/ruby/lib/grpc/version.rb +++ b/src/ruby/lib/grpc/version.rb @@ -14,5 +14,5 @@ # GRPC contains the General RPC module. module GRPC - VERSION = '1.14.0.dev' + VERSION = '1.15.0.dev' end diff --git a/src/ruby/tools/version.rb b/src/ruby/tools/version.rb index dad09bb965e..1b5e2c362be 100644 --- a/src/ruby/tools/version.rb +++ b/src/ruby/tools/version.rb @@ -14,6 +14,6 @@ module GRPC module Tools - VERSION = '1.14.0.dev' + VERSION = '1.15.0.dev' end end diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index ae249e73861..ccb69a8ebcb 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION = '1.14.0.dev0' +VERSION = '1.15.0.dev0' diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 322ab5eb983..2f06bda0168 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.14.0-dev +PROJECT_NUMBER = 1.15.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index ba322a90a54..a46ebe61976 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.14.0-dev +PROJECT_NUMBER = 1.15.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a From ad371f802d35686ad6776cc01d1287275f6dd9d6 Mon Sep 17 00:00:00 2001 From: Brent Shaffer Date: Thu, 19 Jul 2018 16:52:46 -0700 Subject: [PATCH 03/25] fixes subpackage issue with loading composer --- src/php/lib/Grpc/BaseStub.php | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/php/lib/Grpc/BaseStub.php b/src/php/lib/Grpc/BaseStub.php index ecb419ac8f8..fe81e377610 100644 --- a/src/php/lib/Grpc/BaseStub.php +++ b/src/php/lib/Grpc/BaseStub.php @@ -83,10 +83,11 @@ class BaseStub } private static function updateOpts($opts) { - $package_config = json_decode( - file_get_contents(dirname(__FILE__).'/../../composer.json'), - true - ); + if (!file_exists($composerFile = __DIR__.'/../../composer.json')) { + // for grpc/grpc-php subpackage + $composerFile = __DIR__.'/../composer.json'; + } + $package_config = json_decode(file_get_contents($composerFile), true); if (!empty($opts['grpc.primary_user_agent'])) { $opts['grpc.primary_user_agent'] .= ' '; } else { From 632eb7acaac3ef55ddd9256a68bfabce5157e6ab Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Mon, 23 Jul 2018 08:40:06 -0700 Subject: [PATCH 04/25] Add protoc packages to nightly builds --- .../linux/grpc_publish_packages.sh | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tools/internal_ci/linux/grpc_publish_packages.sh b/tools/internal_ci/linux/grpc_publish_packages.sh index 24bb4880ed8..ebd19c59542 100644 --- a/tools/internal_ci/linux/grpc_publish_packages.sh +++ b/tools/internal_ci/linux/grpc_publish_packages.sh @@ -39,12 +39,31 @@ mkdir -p "$LOCAL_BUILD_ROOT" find "$INPUT_ARTIFACTS" -type f +# protoc Plugins +PROTOC_PLUGINS_ZIPPED_PACKAGES=$(mktemp -d) +for zip_dir in protoc_windows_{x86,x64} +do + zip -jr "$PROTOC_PLUGINS_ZIPPED_PACKAGES/$zip_dir.zip" "$INPUT_ARTIFACTS/$zip_dir/"* +done +for tar_dir in protoc_{linux,macos}_{x86,x64} +do + chmod +x "$INPUT_ARTIFACTS/$tar_dir"/* + tar -cvzf "$PROTOC_PLUGINS_ZIPPED_PACKAGES/$tar_dir.tar.gz" -C "$INPUT_ARTIFACTS/$tar_dir" . +done + +PROTOC_PACKAGES=( + "$PROTOC_PLUGINS_ZIPPED_PACKAGES"/protoc_windows_{x86,x64}.zip + "$PROTOC_PLUGINS_ZIPPED_PACKAGES"/protoc_{linux,macos}_{x86,x64}.tar.gz +) + +# C# UNZIPPED_CSHARP_PACKAGES=$(mktemp -d) unzip "$INPUT_ARTIFACTS/csharp_nugets_windows_dotnetcli.zip" -d "$UNZIPPED_CSHARP_PACKAGES" CSHARP_PACKAGES=( "$UNZIPPED_CSHARP_PACKAGES"/* ) +# Python PYTHON_PACKAGES=( "$INPUT_ARTIFACTS"/grpcio-[0-9]*.tar.gz "$INPUT_ARTIFACTS"/grpcio-[0-9]*.whl @@ -59,10 +78,12 @@ PYTHON_PACKAGES=( "$INPUT_ARTIFACTS"/grpcio-testing-[0-9]*.tar.gz ) +# PHP PHP_PACKAGES=( "$INPUT_ARTIFACTS"/grpc-[0-9]*.tgz ) +# Ruby RUBY_PACKAGES=( "$INPUT_ARTIFACTS"/grpc-[0-9]*.gem "$INPUT_ARTIFACTS"/grpc-tools-[0-9]*.gem @@ -100,6 +121,7 @@ EOF EOF + for pkg in "${PROTOC_PACKAGES[@]}"; do add_to_manifest protoc "$pkg"; done for pkg in "${CSHARP_PACKAGES[@]}"; do add_to_manifest csharp "$pkg"; done for pkg in "${PHP_PACKAGES[@]}"; do add_to_manifest php "$pkg"; done for pkg in "${PYTHON_PACKAGES[@]}"; do add_to_manifest python "$pkg"; done From 536c4aadf3f5ebb84281c6f1ed05d3e649aac18f Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Mon, 23 Jul 2018 08:44:43 -0700 Subject: [PATCH 05/25] Add template version number to build manifest XML files --- tools/internal_ci/linux/grpc_publish_packages.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/internal_ci/linux/grpc_publish_packages.sh b/tools/internal_ci/linux/grpc_publish_packages.sh index ebd19c59542..2439e40c7e5 100644 --- a/tools/internal_ci/linux/grpc_publish_packages.sh +++ b/tools/internal_ci/linux/grpc_publish_packages.sh @@ -111,7 +111,7 @@ EOF cat < - + gRPC https://github.com/grpc/grpc From 775f31e1746278ae457d496cefc6ea8323ad855f Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Mon, 23 Jul 2018 09:18:04 -0700 Subject: [PATCH 06/25] Put different Python packages in different directories --- .../linux/grpc_publish_packages.sh | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/tools/internal_ci/linux/grpc_publish_packages.sh b/tools/internal_ci/linux/grpc_publish_packages.sh index 2439e40c7e5..cab5cf05146 100644 --- a/tools/internal_ci/linux/grpc_publish_packages.sh +++ b/tools/internal_ci/linux/grpc_publish_packages.sh @@ -64,17 +64,23 @@ CSHARP_PACKAGES=( ) # Python -PYTHON_PACKAGES=( +PYTHON_GRPCIO_PACKAGES=( "$INPUT_ARTIFACTS"/grpcio-[0-9]*.tar.gz "$INPUT_ARTIFACTS"/grpcio-[0-9]*.whl "$INPUT_ARTIFACTS"/python_linux_extra_arm*/grpcio-[0-9]*.whl - +) +PYTHON_GRPCIO_TOOLS_PACKAGES=( "$INPUT_ARTIFACTS"/grpcio-tools-[0-9]*.tar.gz "$INPUT_ARTIFACTS"/grpcio_tools-[0-9]*.whl "$INPUT_ARTIFACTS"/python_linux_extra_arm*/grpcio_tools-[0-9]*.whl - +) +PYTHON_GRPCIO_HEALTH_CHECKING_PACKAGES=( "$INPUT_ARTIFACTS"/grpcio-health-checking-[0-9]*.tar.gz +) +PYTHON_GRPCIO_REFLECTION_PACKAGES=( "$INPUT_ARTIFACTS"/grpcio-reflection-[0-9]*.tar.gz +) +PYTHON_GRPCIO_TESTING_PACKAGES=( "$INPUT_ARTIFACTS"/grpcio-testing-[0-9]*.tar.gz ) @@ -92,17 +98,18 @@ RUBY_PACKAGES=( function add_to_manifest() { local artifact_type=$1 local artifact_file=$2 + local artifact_prefix=$3 local artifact_name artifact_name=$(basename "$artifact_file") local artifact_sha256 artifact_sha256=$(openssl sha256 -r "$artifact_file" | cut -d " " -f 1) - local artifact_target=$LOCAL_BUILD_ROOT/$artifact_type + local artifact_target=$LOCAL_BUILD_ROOT/$artifact_type/$artifact_prefix mkdir -p "$artifact_target" cp "$artifact_file" "$artifact_target" cat < EOF } @@ -124,7 +131,11 @@ EOF for pkg in "${PROTOC_PACKAGES[@]}"; do add_to_manifest protoc "$pkg"; done for pkg in "${CSHARP_PACKAGES[@]}"; do add_to_manifest csharp "$pkg"; done for pkg in "${PHP_PACKAGES[@]}"; do add_to_manifest php "$pkg"; done - for pkg in "${PYTHON_PACKAGES[@]}"; do add_to_manifest python "$pkg"; done + for pkg in "${PYTHON_GRPCIO_PACKAGES[@]}"; do add_to_manifest python "$pkg" grpcio/; done + for pkg in "${PYTHON_GRPCIO_TOOLS_PACKAGES[@]}"; do add_to_manifest python "$pkg" grpcio-tools/; done + for pkg in "${PYTHON_GRPCIO_HEALTH_CHECKING_PACKAGES[@]}"; do add_to_manifest python "$pkg" grpcio-health-checking/; done + for pkg in "${PYTHON_GRPCIO_REFLECTION_PACKAGES[@]}"; do add_to_manifest python "$pkg" grpcio-reflection/; done + for pkg in "${PYTHON_GRPCIO_TESTING_PACKAGES[@]}"; do add_to_manifest python "$pkg" grpcio-testing/; done for pkg in "${RUBY_PACKAGES[@]}"; do add_to_manifest ruby "$pkg"; done cat < Date: Mon, 23 Jul 2018 09:58:51 -0700 Subject: [PATCH 07/25] Add file size to manifest files --- tools/internal_ci/linux/grpc_publish_packages.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/internal_ci/linux/grpc_publish_packages.sh b/tools/internal_ci/linux/grpc_publish_packages.sh index cab5cf05146..bf0c313599b 100644 --- a/tools/internal_ci/linux/grpc_publish_packages.sh +++ b/tools/internal_ci/linux/grpc_publish_packages.sh @@ -101,6 +101,8 @@ function add_to_manifest() { local artifact_prefix=$3 local artifact_name artifact_name=$(basename "$artifact_file") + local artifact_size + artifact_size=$(stat -c%s "$artifact_file") local artifact_sha256 artifact_sha256=$(openssl sha256 -r "$artifact_file" | cut -d " " -f 1) local artifact_target=$LOCAL_BUILD_ROOT/$artifact_type/$artifact_prefix @@ -110,6 +112,7 @@ function add_to_manifest() { EOF } @@ -144,6 +147,7 @@ EOF EOF }> "$LOCAL_BUILD_INDEX" +LOCAL_BUILD_INDEX_SIZE=$(stat -c%s "$LOCAL_BUILD_INDEX") LOCAL_BUILD_INDEX_SHA256=$(openssl sha256 -r "$LOCAL_BUILD_INDEX" | cut -d " " -f 1) OLD_INDEX=$(mktemp) @@ -166,6 +170,7 @@ gsutil cp "$GCS_INDEX" "$OLD_INDEX" branch='$BUILD_BRANCH_NAME' commit='$BUILD_GIT_COMMIT' path='$GCS_ARCHIVE_PREFIX$BUILD_RELPATH$INDEX_FILENAME' + size='$LOCAL_BUILD_INDEX_SIZE' sha256='$LOCAL_BUILD_INDEX_SHA256' /> EOF tail --lines=+5 "$OLD_INDEX" From 92e0a70ec64f61918faab50a14b0587cf1aaaa3a Mon Sep 17 00:00:00 2001 From: Alex Polcyn Date: Sat, 21 Jul 2018 07:38:34 +0000 Subject: [PATCH 08/25] Make QPS server builder customizeabled --- CMakeLists.txt | 1 + Makefile | 3 ++ build.yaml | 2 + grpc.gyp | 1 + test/cpp/qps/BUILD | 6 +++ test/cpp/qps/qps_server_builder.cc | 45 ++++++++++++++++++ test/cpp/qps/qps_server_builder.h | 46 +++++++++++++++++++ test/cpp/qps/qps_worker.cc | 9 ++-- test/cpp/qps/server_async.cc | 15 +++--- test/cpp/qps/server_sync.cc | 13 +++--- .../generated/sources_and_headers.json | 3 ++ 11 files changed, 127 insertions(+), 17 deletions(-) create mode 100644 test/cpp/qps/qps_server_builder.cc create mode 100644 test/cpp/qps/qps_server_builder.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 75f3a2f3cdf..93991de4224 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5164,6 +5164,7 @@ add_library(qps test/cpp/qps/client_sync.cc test/cpp/qps/driver.cc test/cpp/qps/parse_json.cc + test/cpp/qps/qps_server_builder.cc test/cpp/qps/qps_worker.cc test/cpp/qps/report.cc test/cpp/qps/server_async.cc diff --git a/Makefile b/Makefile index 309187969cc..f829b3abd08 100644 --- a/Makefile +++ b/Makefile @@ -7438,6 +7438,7 @@ LIBQPS_SRC = \ test/cpp/qps/client_sync.cc \ test/cpp/qps/driver.cc \ test/cpp/qps/parse_json.cc \ + test/cpp/qps/qps_server_builder.cc \ test/cpp/qps/qps_worker.cc \ test/cpp/qps/report.cc \ test/cpp/qps/server_async.cc \ @@ -7493,6 +7494,7 @@ $(OBJDIR)/$(CONFIG)/test/cpp/qps/client_async.o: $(GENDIR)/src/proto/grpc/testin $(OBJDIR)/$(CONFIG)/test/cpp/qps/client_sync.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/qps/driver.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/qps/parse_json.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/qps/qps_server_builder.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/qps/qps_worker.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/qps/report.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/qps/server_async.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.grpc.pb.cc @@ -24691,6 +24693,7 @@ test/cpp/qps/client_async.cc: $(OPENSSL_DEP) test/cpp/qps/client_sync.cc: $(OPENSSL_DEP) test/cpp/qps/driver.cc: $(OPENSSL_DEP) test/cpp/qps/parse_json.cc: $(OPENSSL_DEP) +test/cpp/qps/qps_server_builder.cc: $(OPENSSL_DEP) test/cpp/qps/qps_worker.cc: $(OPENSSL_DEP) test/cpp/qps/report.cc: $(OPENSSL_DEP) test/cpp/qps/server_async.cc: $(OPENSSL_DEP) diff --git a/build.yaml b/build.yaml index 2ff98f12bc6..c76aa464354 100644 --- a/build.yaml +++ b/build.yaml @@ -1963,6 +1963,7 @@ libs: - test/cpp/qps/histogram.h - test/cpp/qps/interarrival.h - test/cpp/qps/parse_json.h + - test/cpp/qps/qps_server_builder.h - test/cpp/qps/qps_worker.h - test/cpp/qps/report.h - test/cpp/qps/server.h @@ -1981,6 +1982,7 @@ libs: - test/cpp/qps/client_sync.cc - test/cpp/qps/driver.cc - test/cpp/qps/parse_json.cc + - test/cpp/qps/qps_server_builder.cc - test/cpp/qps/qps_worker.cc - test/cpp/qps/report.cc - test/cpp/qps/server_async.cc diff --git a/grpc.gyp b/grpc.gyp index 7cbd9eded78..e1485efa058 100644 --- a/grpc.gyp +++ b/grpc.gyp @@ -1718,6 +1718,7 @@ 'test/cpp/qps/client_sync.cc', 'test/cpp/qps/driver.cc', 'test/cpp/qps/parse_json.cc', + 'test/cpp/qps/qps_server_builder.cc', 'test/cpp/qps/qps_worker.cc', 'test/cpp/qps/report.cc', 'test/cpp/qps/server_async.cc', diff --git a/test/cpp/qps/BUILD b/test/cpp/qps/BUILD index e7d093c71a4..b958c75fc7c 100644 --- a/test/cpp/qps/BUILD +++ b/test/cpp/qps/BUILD @@ -34,11 +34,13 @@ grpc_cc_library( "qps_worker.cc", "server_async.cc", "server_sync.cc", + "qps_server_builder.cc", ], hdrs = [ "client.h", "qps_worker.h", "server.h", + "qps_server_builder.h", ], deps = [ ":histogram", @@ -55,6 +57,10 @@ grpc_cc_library( "//test/core/util:gpr_test_util", "//test/core/util:grpc_test_util", "//test/cpp/util:test_util", + "//test/cpp/util:test_config", + ], + external_deps = [ + "gflags", ], ) diff --git a/test/cpp/qps/qps_server_builder.cc b/test/cpp/qps/qps_server_builder.cc new file mode 100644 index 00000000000..5fbc682b756 --- /dev/null +++ b/test/cpp/qps/qps_server_builder.cc @@ -0,0 +1,45 @@ +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "qps_server_builder.h" + +using grpc::ServerBuilder; + +namespace grpc { +namespace testing { + +namespace { +std::unique_ptr DefaultCreateQpsServerBuilder() { + return std::unique_ptr(new ServerBuilder()); +} + +std::function()> g_create_qps_server_builder = + DefaultCreateQpsServerBuilder; +} // namespace + +std::unique_ptr CreateQpsServerBuilder() { + return g_create_qps_server_builder(); +} + +void SetCreateQpsServerBuilderFunc( + std::function()> create_qps_server_builder) { + g_create_qps_server_builder = std::move(create_qps_server_builder); +} + +} // namespace testing +} // namespace grpc diff --git a/test/cpp/qps/qps_server_builder.h b/test/cpp/qps/qps_server_builder.h new file mode 100644 index 00000000000..98f9fa72e97 --- /dev/null +++ b/test/cpp/qps/qps_server_builder.h @@ -0,0 +1,46 @@ +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPC_QPS_SERVER_BUILDER_H +#define GRPC_QPS_SERVER_BUILDER_H + +#include +#include + +#include + +namespace grpc { +namespace testing { + +// CreateQpsServerBuilder creates a new ServerBuilder. +// This uses the "create ServerBuilder" func that was set +// in SetCreateQpsServerBuilderFunc if one has been set, +// otherwise, this defaults to creating a new ServerBuilder +// with only its default constructor. +std::unique_ptr CreateQpsServerBuilder(); + +// SetCreateQpsServerBuilderFunc sets a function to use to create new +// ServerBuilders in "CreateQpsServerBuilder". It can be used to modify options +// that the server is built with. +void SetCreateQpsServerBuilderFunc( + std::function()>); + +} // namespace testing +} // namespace grpc + +#endif // GRPC_QPS_SERVER_BUILDER_H diff --git a/test/cpp/qps/qps_worker.cc b/test/cpp/qps/qps_worker.cc index d3f03804743..7ddf3c1cf3b 100644 --- a/test/cpp/qps/qps_worker.cc +++ b/test/cpp/qps/qps_worker.cc @@ -39,6 +39,7 @@ #include "test/core/util/grpc_profiler.h" #include "test/core/util/histogram.h" #include "test/cpp/qps/client.h" +#include "test/cpp/qps/qps_server_builder.h" #include "test/cpp/qps/server.h" #include "test/cpp/util/create_test_channel.h" #include "test/cpp/util/test_credentials_provider.h" @@ -272,18 +273,18 @@ QpsWorker::QpsWorker(int driver_port, int server_port, impl_.reset(new WorkerServiceImpl(server_port, this)); gpr_atm_rel_store(&done_, static_cast(0)); - ServerBuilder builder; + std::unique_ptr builder = CreateQpsServerBuilder(); if (driver_port >= 0) { char* server_address = nullptr; gpr_join_host_port(&server_address, "::", driver_port); - builder.AddListeningPort( + builder->AddListeningPort( server_address, GetCredentialsProvider()->GetServerCredentials(credential_type)); gpr_free(server_address); } - builder.RegisterService(impl_.get()); + builder->RegisterService(impl_.get()); - server_ = builder.BuildAndStart(); + server_ = builder->BuildAndStart(); } QpsWorker::~QpsWorker() {} diff --git a/test/cpp/qps/server_async.cc b/test/cpp/qps/server_async.cc index 1dfef6cfc17..5cd975cf742 100644 --- a/test/cpp/qps/server_async.cc +++ b/test/cpp/qps/server_async.cc @@ -38,6 +38,7 @@ #include "src/core/lib/surface/completion_queue.h" #include "src/proto/grpc/testing/benchmark_service.grpc.pb.h" #include "test/core/util/test_config.h" +#include "test/cpp/qps/qps_server_builder.h" #include "test/cpp/qps/server.h" namespace grpc { @@ -74,19 +75,19 @@ class AsyncQpsServerTest final : public grpc::testing::Server { ResponseType*)> process_rpc) : Server(config) { - ServerBuilder builder; + std::unique_ptr builder = CreateQpsServerBuilder(); auto port_num = port(); // Negative port number means inproc server, so no listen port needed if (port_num >= 0) { char* server_address = nullptr; gpr_join_host_port(&server_address, "::", port_num); - builder.AddListeningPort(server_address, - Server::CreateServerCredentials(config)); + builder->AddListeningPort(server_address, + Server::CreateServerCredentials(config)); gpr_free(server_address); } - register_service(&builder, &async_service_); + register_service(builder.get(), &async_service_); int num_threads = config.async_server_threads(); if (num_threads <= 0) { // dynamic sizing @@ -97,15 +98,15 @@ class AsyncQpsServerTest final : public grpc::testing::Server { int tpc = std::max(1, config.threads_per_cq()); // 1 if unspecified int num_cqs = (num_threads + tpc - 1) / tpc; // ceiling operator for (int i = 0; i < num_cqs; i++) { - srv_cqs_.emplace_back(builder.AddCompletionQueue()); + srv_cqs_.emplace_back(builder->AddCompletionQueue()); } for (int i = 0; i < num_threads; i++) { cq_.emplace_back(i % srv_cqs_.size()); } - ApplyConfigToBuilder(config, &builder); + ApplyConfigToBuilder(config, builder.get()); - server_ = builder.BuildAndStart(); + server_ = builder->BuildAndStart(); auto process_rpc_bound = std::bind(process_rpc, config.payload_config(), std::placeholders::_1, diff --git a/test/cpp/qps/server_sync.cc b/test/cpp/qps/server_sync.cc index b8facf9b569..2e63f5ec867 100644 --- a/test/cpp/qps/server_sync.cc +++ b/test/cpp/qps/server_sync.cc @@ -27,6 +27,7 @@ #include "src/core/lib/gpr/host_port.h" #include "src/proto/grpc/testing/benchmark_service.grpc.pb.h" +#include "test/cpp/qps/qps_server_builder.h" #include "test/cpp/qps/server.h" #include "test/cpp/qps/usage_timer.h" @@ -154,23 +155,23 @@ class BenchmarkServiceImpl final : public BenchmarkService::Service { class SynchronousServer final : public grpc::testing::Server { public: explicit SynchronousServer(const ServerConfig& config) : Server(config) { - ServerBuilder builder; + std::unique_ptr builder = CreateQpsServerBuilder(); auto port_num = port(); // Negative port number means inproc server, so no listen port needed if (port_num >= 0) { char* server_address = nullptr; gpr_join_host_port(&server_address, "::", port_num); - builder.AddListeningPort(server_address, - Server::CreateServerCredentials(config)); + builder->AddListeningPort(server_address, + Server::CreateServerCredentials(config)); gpr_free(server_address); } - ApplyConfigToBuilder(config, &builder); + ApplyConfigToBuilder(config, builder.get()); - builder.RegisterService(&service_); + builder->RegisterService(&service_); - impl_ = builder.BuildAndStart(); + impl_ = builder->BuildAndStart(); } std::shared_ptr InProcessChannel( diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index bf3ddb42c92..072402b2cf6 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -7761,6 +7761,7 @@ "test/cpp/qps/histogram.h", "test/cpp/qps/interarrival.h", "test/cpp/qps/parse_json.h", + "test/cpp/qps/qps_server_builder.h", "test/cpp/qps/qps_worker.h", "test/cpp/qps/report.h", "test/cpp/qps/server.h", @@ -7782,6 +7783,8 @@ "test/cpp/qps/interarrival.h", "test/cpp/qps/parse_json.cc", "test/cpp/qps/parse_json.h", + "test/cpp/qps/qps_server_builder.cc", + "test/cpp/qps/qps_server_builder.h", "test/cpp/qps/qps_worker.cc", "test/cpp/qps/qps_worker.h", "test/cpp/qps/report.cc", From cebab02bf4ee3f8d38609d4c76030607073ca179 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 23 Jul 2018 19:12:29 +0200 Subject: [PATCH 09/25] C# native libraries should support Android 4.4+ (=api level 19) --- src/csharp/experimental/build_native_ext_for_android.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/csharp/experimental/build_native_ext_for_android.sh b/src/csharp/experimental/build_native_ext_for_android.sh index 5687a43a4e2..6518e03c53e 100755 --- a/src/csharp/experimental/build_native_ext_for_android.sh +++ b/src/csharp/experimental/build_native_ext_for_android.sh @@ -36,6 +36,7 @@ cd build # ANDROID_ABI in ('arm64-v8a', 'armeabi-v7a') # e.g. ANDROID_ABI=armeabi-v7a +# android-19 corresponds to Kitkat 4.4 ${ANDROID_SDK_CMAKE} ../.. \ -DCMAKE_TOOLCHAIN_FILE="${ANDROID_NDK_PATH}/build/cmake/android.toolchain.cmake" \ -DCMAKE_ANDROID_NDK="${ANDROID_NDK_PATH}" \ @@ -44,7 +45,7 @@ ${ANDROID_SDK_CMAKE} ../.. \ -DRUN_HAVE_STD_REGEX=0 \ -DRUN_HAVE_STEADY_CLOCK=0 \ -DCMAKE_BUILD_TYPE=Release \ - -DANDROID_PLATFORM=android-28 \ + -DANDROID_PLATFORM=android-19 \ -DANDROID_ABI="${ANDROID_ABI}" \ -DANDROID_NDK="${ANDROID_NDK_PATH}" From 0b907fc9d1cc37a69ab7877567154be8ab757e7f Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 23 Jul 2018 19:13:41 +0200 Subject: [PATCH 10/25] improve xamarin platform detection --- src/csharp/Grpc.Core/Internal/PlatformApis.cs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/csharp/Grpc.Core/Internal/PlatformApis.cs b/src/csharp/Grpc.Core/Internal/PlatformApis.cs index 6c4ee0bdb79..c501aa89fb6 100644 --- a/src/csharp/Grpc.Core/Internal/PlatformApis.cs +++ b/src/csharp/Grpc.Core/Internal/PlatformApis.cs @@ -33,8 +33,9 @@ namespace Grpc.Core.Internal internal static class PlatformApis { const string UnityEngineApplicationClassName = "UnityEngine.Application, UnityEngine"; - const string XamarinAndroidActivityClassName = "Android.App.Activity, Mono.Android"; - const string XamariniOSEnumClassName = "Mono.CSharp.Enum, Mono.CSharp"; + const string XamarinAndroidObjectClassName = "Java.Lang.Object, Mono.Android"; + const string XamarinIOSObjectClassName = "Foundation.NSObject, Xamarin.iOS"; + static readonly bool isLinux; static readonly bool isMacOSX; static readonly bool isWindows; @@ -42,7 +43,7 @@ namespace Grpc.Core.Internal static readonly bool isNetCore; static readonly bool isUnity; static readonly bool isXamarin; - static readonly bool isXamariniOS; + static readonly bool isXamarinIOS; static readonly bool isXamarinAndroid; static PlatformApis() @@ -63,9 +64,9 @@ namespace Grpc.Core.Internal #endif isMono = Type.GetType("Mono.Runtime") != null; isUnity = Type.GetType(UnityEngineApplicationClassName) != null; - isXamariniOS = Type.GetType(XamariniOSEnumClassName) != null; - isXamarinAndroid = Type.GetType(XamarinAndroidActivityClassName) != null; - isXamarin = isXamariniOS || isXamarinAndroid; + isXamarinIOS = Type.GetType(XamarinIOSObjectClassName) != null; + isXamarinAndroid = Type.GetType(XamarinAndroidObjectClassName) != null; + isXamarin = isXamarinIOS || isXamarinAndroid; } public static bool IsLinux @@ -108,9 +109,9 @@ namespace Grpc.Core.Internal /// /// true if running on Xamarin.iOS, false otherwise. /// - public static bool IsXamariniOS + public static bool IsXamarinIOS { - get { return isXamariniOS; } + get { return isXamarinIOS; } } /// From efe02c0dfb575ec4c0e13fd152c01c70d45df995 Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Mon, 23 Jul 2018 10:34:37 -0700 Subject: [PATCH 11/25] Generate directory index HTML file --- .../linux/grpc_publish_packages.sh | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tools/internal_ci/linux/grpc_publish_packages.sh b/tools/internal_ci/linux/grpc_publish_packages.sh index bf0c313599b..fecb9a5e09b 100644 --- a/tools/internal_ci/linux/grpc_publish_packages.sh +++ b/tools/internal_ci/linux/grpc_publish_packages.sh @@ -176,7 +176,55 @@ EOF tail --lines=+5 "$OLD_INDEX" }> "$NEW_INDEX" + +function generate_directory_index() +{ + local target_dir=$1 + local current_directory_name + current_directory_name=$(basename "$target_dir") + cat < + + + + Index of $current_directory_name - packages.grpc.io + + + +

Index of $current_directory_name

+
    +
  • .
    • +
  • ..
    • +EOF + +( + cd "$target_dir" + find * -maxdepth 0 -type d -print | sort | while read -r line + do + echo "
  • $line/
    • " + done + find * -maxdepth 0 -type f -print | sort | while read -r line + do + echo "
  • $line
    • " + done +) + +cat < + + +EOF +} + # Upload the current build artifacts gsutil -m cp -r "$LOCAL_STAGING_TEMPDIR/${BUILD_RELPATH%%/*}" "$GCS_ARCHIVE_ROOT" +# Upload directory indicies for subdirectories +( + cd "$LOCAL_BUILD_ROOT" + find * -type d | while read -r directory + do + generate_directory_index "$directory" | gsutil -h 'Content-Type:text/html' cp - "$GCS_ARCHIVE_ROOT$BUILD_RELPATH$directory/$INDEX_FILENAME" + done +) # Upload the new /index.xml gsutil -h "Content-Type:application/xml" cp "$NEW_INDEX" "$GCS_INDEX" From 081c27baad81f3f4862b5c89f0bf1bd792d19876 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 23 Jul 2018 22:59:02 +0200 Subject: [PATCH 12/25] add static_csharp rule to Makefile --- Makefile | 2 ++ templates/Makefile.template | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/Makefile b/Makefile index 2b4ed146ccc..bad41975a05 100644 --- a/Makefile +++ b/Makefile @@ -1381,6 +1381,8 @@ static_c: pc_c pc_c_unsecure cache.mk $(LIBDIR)/$(CONFIG)/libaddress_sorting.a static_cxx: pc_cxx pc_cxx_unsecure cache.mk $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc++_cronet.a $(LIBDIR)/$(CONFIG)/libgrpc++_error_details.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpcpp_channelz.a +static_csharp: static_c $(LIBDIR)/$(CONFIG)/libgrpc_csharp_ext.a + shared: shared_c shared_cxx shared_c: pc_c pc_c_unsecure cache.mk $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)address_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc_cronet$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) diff --git a/templates/Makefile.template b/templates/Makefile.template index 628056ed4db..50b81e5f9f2 100644 --- a/templates/Makefile.template +++ b/templates/Makefile.template @@ -921,6 +921,16 @@ % endfor + static_csharp: static_c \ + % for lib in libs: + % if 'Makefile' in lib.get('build_system', ['Makefile']): + % if lib.build == 'all' and lib.language == 'csharp': + $(LIBDIR)/$(CONFIG)/lib${lib.name}.a\ + % endif + % endif + % endfor + + shared: shared_c shared_cxx shared_c: pc_c pc_c_unsecure cache.mk\ From bcf271c416affe9871a43c8583d4295e8d73159a Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Mon, 23 Jul 2018 15:16:32 -0700 Subject: [PATCH 13/25] Allow error strings in final_info to propagate to filters on call destruction --- src/core/lib/channel/channel_stack.h | 2 +- src/core/lib/surface/call.cc | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/core/lib/channel/channel_stack.h b/src/core/lib/channel/channel_stack.h index 7581f937b64..35c3fb01eaa 100644 --- a/src/core/lib/channel/channel_stack.h +++ b/src/core/lib/channel/channel_stack.h @@ -82,7 +82,7 @@ typedef struct { typedef struct { grpc_call_stats stats; grpc_status_code final_status; - const char** error_string; + const char* error_string; } grpc_call_final_info; /* Channel filters specify: diff --git a/src/core/lib/surface/call.cc b/src/core/lib/surface/call.cc index 88e015ce22f..847bb87f7e5 100644 --- a/src/core/lib/surface/call.cc +++ b/src/core/lib/surface/call.cc @@ -561,7 +561,7 @@ static void destroy_call(void* call, grpc_error* error) { } get_final_status(c, set_status_value_directly, &c->final_info.final_status, - nullptr, c->final_info.error_string); + nullptr, &(c->final_info.error_string)); c->final_info.stats.latency = gpr_time_sub(gpr_now(GPR_CLOCK_MONOTONIC), c->start_time); @@ -573,6 +573,7 @@ static void destroy_call(void* call, grpc_error* error) { grpc_call_stack_destroy(CALL_STACK_FROM_CALL(c), &c->final_info, GRPC_CLOSURE_INIT(&c->release_call, release_call, c, grpc_schedule_on_exec_ctx)); + gpr_free(static_cast(const_cast(c->final_info.error_string))); } void grpc_call_ref(grpc_call* c) { gpr_ref(&c->ext_ref); } From e971fa4490bb98e9e35dac7b094204ddbd8a3f80 Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Mon, 23 Jul 2018 18:29:28 -0700 Subject: [PATCH 14/25] Add link to packages.grpc.io --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 1af9ec34f0a..1960f44137c 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,8 @@ For instructions on how to use the language-specific gRPC runtime in your projec You can find per-language quickstart guides and tutorials in [Documentation section on grpc.io website](https://grpc.io/docs/). The code examples are available in the [examples](examples) directory. +Precompiled bleeding-edge package builds of gRPC `master` branch's `HEAD` are uploaded daily to [packages.grpc.io](https://packages.grpc.io). + # To start developing gRPC Contributions are welcome! From 95390f943e8fb944f5f434b58cc3032b59f94eb8 Mon Sep 17 00:00:00 2001 From: Hope Casey-Allen Date: Thu, 19 Jul 2018 14:18:17 -0700 Subject: [PATCH 15/25] Replace atomics in gpr_arena_aloc with a mutex in the slow path --- src/core/lib/gpr/arena.cc | 88 ++++++++++++++++++++++++--------------- 1 file changed, 55 insertions(+), 33 deletions(-) diff --git a/src/core/lib/gpr/arena.cc b/src/core/lib/gpr/arena.cc index 141de69426c..e30b297aeab 100644 --- a/src/core/lib/gpr/arena.cc +++ b/src/core/lib/gpr/arena.cc @@ -25,6 +25,7 @@ #include #include #include +#include #include "src/core/lib/gpr/alloc.h" @@ -36,8 +37,6 @@ #ifdef SIMPLE_ARENA_FOR_DEBUGGING -#include - struct gpr_arena { gpr_mu mu; void** ptrs; @@ -78,14 +77,17 @@ void* gpr_arena_alloc(gpr_arena* arena, size_t size) { // would allow us to use the alignment actually needed by the caller. typedef struct zone { - size_t size_begin; - size_t size_end; - gpr_atm next_atm; + size_t size_begin; // All the space we have set aside for allocations up + // until this zone. + size_t size_end; // size_end = size_begin plus all the space we set aside for + // allocations in zone z itself. + zone* next; } zone; struct gpr_arena { gpr_atm size_so_far; zone initial_zone; + gpr_mu arena_growth_mutex; }; static void* zalloc_aligned(size_t size) { @@ -99,15 +101,17 @@ gpr_arena* gpr_arena_create(size_t initial_size) { gpr_arena* a = static_cast(zalloc_aligned( GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(gpr_arena)) + initial_size)); a->initial_zone.size_end = initial_size; + gpr_mu_init(&a->arena_growth_mutex); return a; } size_t gpr_arena_destroy(gpr_arena* arena) { + gpr_mu_destroy(&arena->arena_growth_mutex); gpr_atm size = gpr_atm_no_barrier_load(&arena->size_so_far); - zone* z = (zone*)gpr_atm_no_barrier_load(&arena->initial_zone.next_atm); + zone* z = arena->initial_zone.next; gpr_free_aligned(arena); while (z) { - zone* next_z = (zone*)gpr_atm_no_barrier_load(&z->next_atm); + zone* next_z = z->next; gpr_free_aligned(z); z = next_z; } @@ -116,37 +120,55 @@ size_t gpr_arena_destroy(gpr_arena* arena) { void* gpr_arena_alloc(gpr_arena* arena, size_t size) { size = GPR_ROUND_UP_TO_ALIGNMENT_SIZE(size); - size_t start = static_cast( + size_t previous_size_of_arena_allocations = static_cast( gpr_atm_no_barrier_fetch_add(&arena->size_so_far, size)); + size_t updated_size_of_arena_allocations = + previous_size_of_arena_allocations + size; zone* z = &arena->initial_zone; - while (start > z->size_end) { - zone* next_z = (zone*)gpr_atm_acq_load(&z->next_atm); - if (next_z == nullptr) { - size_t next_z_size = - static_cast(gpr_atm_no_barrier_load(&arena->size_so_far)); - next_z = static_cast(zalloc_aligned( - GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(zone)) + next_z_size)); - next_z->size_begin = z->size_end; - next_z->size_end = z->size_end + next_z_size; - if (!gpr_atm_rel_cas(&z->next_atm, static_cast(NULL), - (gpr_atm)next_z)) { - gpr_free_aligned(next_z); - next_z = (zone*)gpr_atm_acq_load(&z->next_atm); + // Check to see if the allocation isn't able to end in the initial zone. + // This statement is true only in the uncommon case because of our arena + // sizing historesis (that is, most calls should have a large enough initial + // zone and will not need to grow the arena). + if (updated_size_of_arena_allocations > z->size_end) { + // Find a zone to fit this allocation + gpr_mu_lock(&arena->arena_growth_mutex); + while (updated_size_of_arena_allocations > z->size_end) { + if (z->next == nullptr) { + // Note that we do an extra increment of size_so_far to prevent multiple + // simultaneous callers from stepping on each other. However, this extra + // increment means some space in the arena is wasted. + // So whenever we need to allocate x bytes and there are x - n (where + // n > 0) remaining in the current zone, we will waste x bytes (x - n + // in the current zone and n in the new zone). + previous_size_of_arena_allocations = static_cast( + gpr_atm_no_barrier_fetch_add(&arena->size_so_far, size)); + updated_size_of_arena_allocations = + previous_size_of_arena_allocations + size; + size_t next_z_size = updated_size_of_arena_allocations; + z->next = static_cast(zalloc_aligned( + GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(zone)) + next_z_size)); + z->next->size_begin = z->size_end; + z->next->size_end = z->size_end + next_z_size; } + z = z->next; } - z = next_z; - } - if (start + size > z->size_end) { - return gpr_arena_alloc(arena, size); + gpr_mu_unlock(&arena->arena_growth_mutex); } - GPR_ASSERT(start >= z->size_begin); - GPR_ASSERT(start + size <= z->size_end); - char* ptr = (z == &arena->initial_zone) - ? reinterpret_cast(arena) + - GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(gpr_arena)) - : reinterpret_cast(z) + - GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(zone)); - return ptr + start - z->size_begin; + GPR_ASSERT(previous_size_of_arena_allocations >= z->size_begin); + GPR_ASSERT(updated_size_of_arena_allocations <= z->size_end); + // Skip the first part of the zone, which just contains tracking information. + // For the initial zone, this is the gpr_arena struct and for any other zone, + // it's the zone struct. + char* start_of_allocation_space = + (z == &arena->initial_zone) + ? reinterpret_cast(arena) + + GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(gpr_arena)) + : reinterpret_cast(z) + + GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(zone)); + // previous_size_of_arena_allocations - size_begin is how many bytes have been + // allocated into the current zone + return start_of_allocation_space + previous_size_of_arena_allocations - + z->size_begin; } #endif // SIMPLE_ARENA_FOR_DEBUGGING From cbc2b1f40562b772c61e1448d4fa290587fca256 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 23 Jul 2018 23:13:34 +0200 Subject: [PATCH 16/25] experiment: build grpc_csharp_ext for ios --- .../experimental/build_native_ext_for_ios.sh | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100755 src/csharp/experimental/build_native_ext_for_ios.sh diff --git a/src/csharp/experimental/build_native_ext_for_ios.sh b/src/csharp/experimental/build_native_ext_for_ios.sh new file mode 100755 index 00000000000..518390ea63b --- /dev/null +++ b/src/csharp/experimental/build_native_ext_for_ios.sh @@ -0,0 +1,45 @@ +#!/bin/sh +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Helper script to crosscompile grpc_csharp_ext native extension for Android. + +set -ex + +cd "$(dirname "$0")/../../.." + +# +SDK="iphoneos" +# +ARCH="arm64" + +PATH_AR="$(xcrun --sdk $SDK --find ar)" +PATH_CC="$(xcrun --sdk $SDK --find clang)" +PATH_CXX="$(xcrun --sdk $SDK --find clang++)" + +# TODO(jtattermusch): add -mios-version-min=6.0 and -Wl,ios_version_min=6.0 +CPPFLAGS="-O2 -Wframe-larger-than=16384 -arch $ARCH -isysroot $(xcrun --sdk $SDK --show-sdk-path) -DPB_NO_PACKED_STRUCTS=1" +LDFLAGS="-arch $ARCH -isysroot $(xcrun --sdk $SDK --show-sdk-path)" + +# TODO(jtattermusch): revisit the build arguments +make -j4 static_csharp \ + VALID_CONFIG_ios_$ARCH="1" \ + CC_ios_$ARCH="$PATH_CC" \ + CXX_ios_$ARCH="$PATH_CXX" \ + LD_ios_$ARCH="$PATH_CC" \ + LDXX_ios_$ARCH="$PATH_CXX" \ + CPPFLAGS_ios_$ARCH="$CPPFLAGS" \ + LDFLAGS_ios_$ARCH="$LDFLAGS" \ + DEFINES_ios_$ARCH="NDEBUG" \ + CONFIG="ios_$ARCH" From 5bb8a33eb26a347bc790620552649c957dcc17e5 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 23 Jul 2018 23:14:11 +0200 Subject: [PATCH 17/25] copying files --- src/csharp/Grpc.Core/Grpc.Core.csproj | 12 ++++++++++++ .../Grpc.Core/build/Xamarin.iOS/Grpc.Core.targets | 15 +++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 src/csharp/Grpc.Core/build/Xamarin.iOS/Grpc.Core.targets diff --git a/src/csharp/Grpc.Core/Grpc.Core.csproj b/src/csharp/Grpc.Core/Grpc.Core.csproj index 0da95d203c6..7b0ae46dc18 100755 --- a/src/csharp/Grpc.Core/Grpc.Core.csproj +++ b/src/csharp/Grpc.Core/Grpc.Core.csproj @@ -54,6 +54,14 @@ runtimes/monoandroid/arm64-v8a/libgrpc_csharp_ext.so true + + runtimes/ios/native/libgrpc_csharp_ext.a + true + + + runtimes/ios/native/libgrpc.a + true + build/net45/ true @@ -62,6 +70,10 @@ build/MonoAndroid/ true + + build/Xamarin.iOS/ + true + diff --git a/src/csharp/Grpc.Core/build/Xamarin.iOS/Grpc.Core.targets b/src/csharp/Grpc.Core/build/Xamarin.iOS/Grpc.Core.targets new file mode 100644 index 00000000000..658158f6ea9 --- /dev/null +++ b/src/csharp/Grpc.Core/build/Xamarin.iOS/Grpc.Core.targets @@ -0,0 +1,15 @@ + + + + + + Static + True + + + Static + True + + + + From d9ad1c3f93540572b2336fdf584eed8bc87495b6 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 23 Jul 2018 23:26:56 +0200 Subject: [PATCH 18/25] building ios artifact --- tools/run_tests/artifacts/artifact_targets.py | 6 +++++ .../artifacts/build_artifact_csharp_ios.sh | 23 +++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100755 tools/run_tests/artifacts/build_artifact_csharp_ios.sh diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index 4500b220e97..3a1142dfee6 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -234,6 +234,11 @@ class CSharpExtArtifact: environ={ 'ANDROID_ABI': self.arch_abi }) + elif self.arch == 'ios': + return create_jobspec( + self.name, + ['tools/run_tests/artifacts/build_artifact_csharp_ios.sh'], + use_workspace=True) elif self.platform == 'windows': cmake_arch_option = 'Win32' if self.arch == 'x86' else self.arch return create_jobspec( @@ -356,6 +361,7 @@ def targets(): ] + [ CSharpExtArtifact('linux', 'android', arch_abi='arm64-v8a'), CSharpExtArtifact('linux', 'android', arch_abi='armeabi-v7a'), + CSharpExtArtifact('macos', 'ios'), PythonArtifact('linux', 'x86', 'cp27-cp27m'), PythonArtifact('linux', 'x86', 'cp27-cp27mu'), PythonArtifact('linux', 'x86', 'cp34-cp34m'), diff --git a/tools/run_tests/artifacts/build_artifact_csharp_ios.sh b/tools/run_tests/artifacts/build_artifact_csharp_ios.sh new file mode 100755 index 00000000000..8b3c7a17b38 --- /dev/null +++ b/tools/run_tests/artifacts/build_artifact_csharp_ios.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Copyright 2016 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +cd "$(dirname "$0")/../../.." + +src/csharp/experimental/build_native_ext_for_ios.sh + +mkdir -p "${ARTIFACTS_OUT}" +cp libs/ios_arm64/libgrpc_csharp_ext.a libs/ios_arm64/libgrpc.a "${ARTIFACTS_OUT}" From 756006f61113b0257d4f7a71f3ea994fbc53b1a3 Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Tue, 24 Jul 2018 01:11:53 -0700 Subject: [PATCH 19/25] Duplicate grpc_build_artifacts_extra job --- .../grpc_build_artifacts_extra_release.cfg | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 tools/internal_ci/linux/grpc_build_artifacts_extra_release.cfg diff --git a/tools/internal_ci/linux/grpc_build_artifacts_extra_release.cfg b/tools/internal_ci/linux/grpc_build_artifacts_extra_release.cfg new file mode 100644 index 00000000000..619e3ea3a97 --- /dev/null +++ b/tools/internal_ci/linux/grpc_build_artifacts_extra_release.cfg @@ -0,0 +1,26 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_build_artifacts_extra.sh" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} From 527d87add53b0ac7f13799facf0115c38846a23d Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 24 Jul 2018 11:36:25 +0200 Subject: [PATCH 20/25] fix Grpc.Core.csproj --- src/csharp/Grpc.Core/Grpc.Core.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/csharp/Grpc.Core/Grpc.Core.csproj b/src/csharp/Grpc.Core/Grpc.Core.csproj index 7b0ae46dc18..c4db3a3a875 100755 --- a/src/csharp/Grpc.Core/Grpc.Core.csproj +++ b/src/csharp/Grpc.Core/Grpc.Core.csproj @@ -54,11 +54,11 @@ runtimes/monoandroid/arm64-v8a/libgrpc_csharp_ext.so true - + runtimes/ios/native/libgrpc_csharp_ext.a true - + runtimes/ios/native/libgrpc.a true From 5aa2a0e376b22f35babf095ab1bc83078a7bf51b Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 24 Jul 2018 14:47:35 +0200 Subject: [PATCH 21/25] build fat binary for ios --- .../experimental/build_native_ext_for_ios.sh | 65 ++++++++++++------- .../artifacts/build_artifact_csharp_ios.sh | 2 +- 2 files changed, 42 insertions(+), 25 deletions(-) diff --git a/src/csharp/experimental/build_native_ext_for_ios.sh b/src/csharp/experimental/build_native_ext_for_ios.sh index 518390ea63b..69c9cdf021c 100755 --- a/src/csharp/experimental/build_native_ext_for_ios.sh +++ b/src/csharp/experimental/build_native_ext_for_ios.sh @@ -19,27 +19,44 @@ set -ex cd "$(dirname "$0")/../../.." -# -SDK="iphoneos" -# -ARCH="arm64" - -PATH_AR="$(xcrun --sdk $SDK --find ar)" -PATH_CC="$(xcrun --sdk $SDK --find clang)" -PATH_CXX="$(xcrun --sdk $SDK --find clang++)" - -# TODO(jtattermusch): add -mios-version-min=6.0 and -Wl,ios_version_min=6.0 -CPPFLAGS="-O2 -Wframe-larger-than=16384 -arch $ARCH -isysroot $(xcrun --sdk $SDK --show-sdk-path) -DPB_NO_PACKED_STRUCTS=1" -LDFLAGS="-arch $ARCH -isysroot $(xcrun --sdk $SDK --show-sdk-path)" - -# TODO(jtattermusch): revisit the build arguments -make -j4 static_csharp \ - VALID_CONFIG_ios_$ARCH="1" \ - CC_ios_$ARCH="$PATH_CC" \ - CXX_ios_$ARCH="$PATH_CXX" \ - LD_ios_$ARCH="$PATH_CC" \ - LDXX_ios_$ARCH="$PATH_CXX" \ - CPPFLAGS_ios_$ARCH="$CPPFLAGS" \ - LDFLAGS_ios_$ARCH="$LDFLAGS" \ - DEFINES_ios_$ARCH="NDEBUG" \ - CONFIG="ios_$ARCH" +# Usage: build +function build { + SDK="$1" + ARCH="$2" + + PATH_AR="$(xcrun --sdk $SDK --find ar)" + PATH_CC="$(xcrun --sdk $SDK --find clang)" + PATH_CXX="$(xcrun --sdk $SDK --find clang++)" + + # TODO(jtattermusch): add -mios-version-min=6.0 and -Wl,ios_version_min=6.0 + CPPFLAGS="-O2 -Wframe-larger-than=16384 -arch $ARCH -isysroot $(xcrun --sdk $SDK --show-sdk-path) -DPB_NO_PACKED_STRUCTS=1" + LDFLAGS="-arch $ARCH -isysroot $(xcrun --sdk $SDK --show-sdk-path)" + + # TODO(jtattermusch): revisit the build arguments + make -j4 static_csharp \ + VALID_CONFIG_ios_$ARCH="1" \ + CC_ios_$ARCH="$PATH_CC" \ + CXX_ios_$ARCH="$PATH_CXX" \ + LD_ios_$ARCH="$PATH_CC" \ + LDXX_ios_$ARCH="$PATH_CXX" \ + CPPFLAGS_ios_$ARCH="$CPPFLAGS" \ + LDFLAGS_ios_$ARCH="$LDFLAGS" \ + DEFINES_ios_$ARCH="NDEBUG" \ + CONFIG="ios_$ARCH" +} + +# Usage: fatten +function fatten { + LIB_NAME="$1" + + mkdir -p libs/ios + lipo -create -output libs/ios/lib$LIB_NAME.a \ + libs/ios_arm64/lib$LIB_NAME.a \ + libs/ios_x86_64/lib$LIB_NAME.a +} + +build iphoneos arm64 +build iphonesimulator x86_64 + +fatten grpc +fatten grpc_csharp_ext diff --git a/tools/run_tests/artifacts/build_artifact_csharp_ios.sh b/tools/run_tests/artifacts/build_artifact_csharp_ios.sh index 8b3c7a17b38..0d0a9c86da7 100755 --- a/tools/run_tests/artifacts/build_artifact_csharp_ios.sh +++ b/tools/run_tests/artifacts/build_artifact_csharp_ios.sh @@ -20,4 +20,4 @@ cd "$(dirname "$0")/../../.." src/csharp/experimental/build_native_ext_for_ios.sh mkdir -p "${ARTIFACTS_OUT}" -cp libs/ios_arm64/libgrpc_csharp_ext.a libs/ios_arm64/libgrpc.a "${ARTIFACTS_OUT}" +cp libs/ios/libgrpc_csharp_ext.a libs/ios/libgrpc.a "${ARTIFACTS_OUT}" From 20f36144be0b475e85e5772f87f51d42cd355d36 Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Tue, 24 Jul 2018 08:49:10 -0700 Subject: [PATCH 22/25] Add packages.grpc.io/web-assets to repo --- tools/package_hosting/404.html | 1 + tools/package_hosting/build-201807.xsl | 114 +++++++++++++++++++++++++ tools/package_hosting/dirindex.css | 16 ++++ tools/package_hosting/home.xsl | 86 +++++++++++++++++++ tools/package_hosting/style.css | 76 +++++++++++++++++ 5 files changed, 293 insertions(+) create mode 100644 tools/package_hosting/404.html create mode 100644 tools/package_hosting/build-201807.xsl create mode 100644 tools/package_hosting/dirindex.css create mode 100644 tools/package_hosting/home.xsl create mode 100644 tools/package_hosting/style.css diff --git a/tools/package_hosting/404.html b/tools/package_hosting/404.html new file mode 100644 index 00000000000..44d986c4b06 --- /dev/null +++ b/tools/package_hosting/404.html @@ -0,0 +1 @@ +404 Not Found diff --git a/tools/package_hosting/build-201807.xsl b/tools/package_hosting/build-201807.xsl new file mode 100644 index 00000000000..69a190446fa --- /dev/null +++ b/tools/package_hosting/build-201807.xsl @@ -0,0 +1,114 @@ + + + + + + + Artifacts for gRPC Build <xsl:value-of select="@id"/> + + + + + + + + + + + + + + + +
    + Artifacts for gRPC Build +
    +
    +
    + Build: + [invocation]
    + Timestamp: +
    + Branch: + + +
    + Commit: + +
    +
    + +
    +
    + +

    gRPC is a Cloud Native Computing Foundation project. Privacy Policy.

    +

    + Copyright ©  The gRPC Authors

    +
    +
    +
    + + +     + + +

    gRPC protoc Plugins

    + + + + +
    + +

    C#

    + + + + +
    + +

    PHP

    + + + + +
    + +

    Python

    + + + + + +
    + +

    Ruby

    + + + + +
    + +     + + + + + + + + + + + + +     diff --git a/tools/package_hosting/dirindex.css b/tools/package_hosting/dirindex.css new file mode 100644 index 00000000000..6c4b04ea323 --- /dev/null +++ b/tools/package_hosting/dirindex.css @@ -0,0 +1,16 @@ +ul { + list-style-type: none; +} +a{ + text-decoration: none; +} +a:hover { + text-decoration: underline; +} +ul li a { + font-family: 'SF Mono', 'Menlo', 'Monaco', 'Consolas', 'Courier New', Courier, monospace +} +h1 { + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, 'Open Sans', 'Helvetica Neue', sans-serif +} + diff --git a/tools/package_hosting/home.xsl b/tools/package_hosting/home.xsl new file mode 100644 index 00000000000..3f79303b1b7 --- /dev/null +++ b/tools/package_hosting/home.xsl @@ -0,0 +1,86 @@ + + + + + + + gRPC Packages + + + + + + + + + + + + + + +
    + gRPC Packages +
    +
    + + +
    +
    +

    gRPC is a Cloud Native Computing Foundation project. Privacy Policy.

    +

    Copyright © 2018 The gRPC Authors

    +
    + + +     + + +

    Official gRPC Releases

    +

    Commits corresponding to official gRPC release points and release candidates are tagged on GitHub.

    +

    To maximize usability, gRPC supports the standard way of adding dependencies in your language of choice (if there is one). + In most languages, the gRPC runtime comes in form of a package available in your language's package manager.

    +

    For instructions on how to use the language-specific gRPC runtime in your project, please refer to the following:

    + +     + + +

    Daily Builds of master Branch

    +

    gRPC packages are built on a daily basis at the HEAD of the master branch and are archived here.

    +

    + The current document (view source) is an XML feed pointing to the packages as they get built and uploaded. + You can subscribe to this feed and fetch, deploy, and test the precompiled packages with your continuous integration infrastructure. +

    +

    For stable release packages, please consult the above section and the common package manager for your language.

    + + + + + + + + + +
    TimestampCommitBuild ID
    +     + + + + + + + + + +     diff --git a/tools/package_hosting/style.css b/tools/package_hosting/style.css new file mode 100644 index 00000000000..dbd26d50cf5 --- /dev/null +++ b/tools/package_hosting/style.css @@ -0,0 +1,76 @@ +html, body +{ + margin: 0; + font-family: sans-serif; +} + +a, a:visited, a:link, a:active { + color: #2da6b0; + text-decoration: none; +} + +a:hover { + color: #2da6b0; + text-decoration: underline; +} + +#topbar { + background-color: #2da6b0; + height: 60px; + margin:auto; +} + +#topbar .title { + position: relative; + top: 24px; + left: 24px; + color: white; + font-family: sans-serif; + font-weight: bold; +} + +#main { + max-width:1100px; + margin:auto; +} + +#main h2 { + text-align: left; +} + +#main table { + width:100%; + border-collapse: collapse; + font-size: small; + font-family: 'SF Mono', 'Menlo', 'Monaco', 'Courier New', Courier, monospace; +} +#main table tr td { + border: solid black 1px; + padding: 5px; +} + +#main table tr td.hash { + text-align: right; + border-left: none; + font-size: x-small; +} + +#main table tr td.name { + text-align: left; + border-right: none; +} + +p.description +{ + font-size: smaller; +} + +#metadata { + margin-top: 15px; + padding: 15px; + font-family: 'SF Mono', 'Menlo', 'Monaco', 'Courier New', Courier, monospace; +} + +#metadata span.fieldname { + font-family: sans-serif; +} \ No newline at end of file From 628c0b5bed15e3a71c63f6fd6bd442d3317c0dc3 Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Tue, 24 Jul 2018 08:53:42 -0700 Subject: [PATCH 23/25] Add upload script for web-assets --- tools/package_hosting/upload_web_assets.sh | 30 ++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100755 tools/package_hosting/upload_web_assets.sh diff --git a/tools/package_hosting/upload_web_assets.sh b/tools/package_hosting/upload_web_assets.sh new file mode 100755 index 00000000000..dcf258e38eb --- /dev/null +++ b/tools/package_hosting/upload_web_assets.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +cd "$(dirname "$0")" + +GCS_WEB_ASSETS=gs://packages.grpc.io/web-assets/ + +WEB_ASSETS=( + 404.html + build-201807.xsl + dirindex.css + home.xsl + style.css +) + +gsutil -m cp "${WEB_ASSETS[@]}" "$GCS_WEB_ASSETS" From 056d061b3138ed26afe3132ed0d3f7d900d4735b Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 24 Jul 2018 19:29:04 +0200 Subject: [PATCH 24/25] Fix license year --- tools/run_tests/artifacts/build_artifact_csharp_ios.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/artifacts/build_artifact_csharp_ios.sh b/tools/run_tests/artifacts/build_artifact_csharp_ios.sh index 0d0a9c86da7..c902a45db7d 100755 --- a/tools/run_tests/artifacts/build_artifact_csharp_ios.sh +++ b/tools/run_tests/artifacts/build_artifact_csharp_ios.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2016 The gRPC Authors +# Copyright 2018 The gRPC Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 04fda25e6e2b6ecfbfd2f8df4db051fd184d0f3d Mon Sep 17 00:00:00 2001 From: Adele Zhou Date: Tue, 24 Jul 2018 12:11:16 -0700 Subject: [PATCH 25/25] Use n1-highmem-2 for most tests. --- third_party/toolchains/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/toolchains/BUILD b/third_party/toolchains/BUILD index 29fc4fdaf3c..02cd87a7b9b 100644 --- a/third_party/toolchains/BUILD +++ b/third_party/toolchains/BUILD @@ -42,7 +42,7 @@ platform( } properties: { name: "gceMachineType" # Small machines for majority of tests. - value: "n1-standard-1" + value: "n1-highmem-2" } properties: { name: "gceMachineType_LARGE" # Large machines for a small set of resource-consuming tests such as combiner_tests under TSAN.