From fda615860069c46b16e8350e1ce20a3f7b3f482a Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 19 Jan 2022 18:28:27 +0100 Subject: [PATCH] Build ruby artifacts in parallel (#28243) * build ruby artifacts in parallel * fine tune grpc_distribtests_ruby.sh parallelism * fine tune linux/grpc_build_artifacts.sh parallelism * cleanup in bundle_install_wrapper * address review comments --- Rakefile | 66 +++++++++++++++---- .../prepare_build_linux_ruby_artifact_rc | 46 +++++++++++++ .../internal_ci/linux/grpc_build_artifacts.sh | 10 +-- .../internal_ci/linux/grpc_build_packages.sh | 7 -- tools/internal_ci/linux/grpc_distribtests.sh | 7 -- .../linux/grpc_distribtests_ruby.sh | 12 ++-- tools/run_tests/artifacts/artifact_targets.py | 31 +++++---- .../artifacts/build_artifact_ruby.sh | 33 +++++----- .../helper_scripts/bundle_install_wrapper.sh | 9 ++- 9 files changed, 150 insertions(+), 71 deletions(-) create mode 100644 tools/internal_ci/helper_scripts/prepare_build_linux_ruby_artifact_rc diff --git a/Rakefile b/Rakefile index 54cc694b3ee..07e738f2113 100755 --- a/Rakefile +++ b/Rakefile @@ -78,12 +78,26 @@ namespace :suite do end end -desc 'Build the Windows gRPC DLLs for Ruby' -task 'dlls' do +desc 'Build the Windows gRPC DLLs for Ruby. The argument contains the list of platforms for which to build dll. Empty placeholder files will be created for platforms that were not selected.' +task 'dlls', [:plat] do |t, args| grpc_config = ENV['GRPC_CONFIG'] || 'opt' verbose = ENV['V'] || '0' # use env variable to set artifact build paralellism nproc_override = ENV['GRPC_RUBY_BUILD_PROCS'] || `nproc`.strip + plat_list = args[:plat] + + build_configs = [] + w64 = { cross: 'x86_64-w64-mingw32', out: 'grpc_c.64.ruby', platform: 'x64-mingw32' } + w32 = { cross: 'i686-w64-mingw32', out: 'grpc_c.32.ruby', platform: 'x86-mingw32' } + [w64, w32].each do |config| + if plat_list.include?(config[:platform]) + # build the DLL (as grpc_c.*.ruby) + build_configs.append(config) + else + # create an empty grpc_c.*.ruby file as a placeholder + FileUtils.touch config[:out] + end + end env = 'CPPFLAGS="-D_WIN32_WINNT=0x600 -DNTDDI_VERSION=0x06000000 -DUNICODE -D_UNICODE -Wno-unused-variable -Wno-unused-result -DCARES_STATICLIB -Wno-error=conversion -Wno-sign-compare -Wno-parentheses -Wno-format -DWIN32_LEAN_AND_MEAN" ' env += 'CFLAGS="-Wno-incompatible-pointer-types" ' @@ -99,10 +113,7 @@ task 'dlls' do out = GrpcBuildConfig::CORE_WINDOWS_DLL - w64 = { cross: 'x86_64-w64-mingw32', out: 'grpc_c.64.ruby', platform: 'x64-mingw32' } - w32 = { cross: 'i686-w64-mingw32', out: 'grpc_c.32.ruby', platform: 'x86-mingw32' } - - [ w64, w32 ].each do |opt| + build_configs.each do |opt| env_comp = "CC=#{opt[:cross]}-gcc " env_comp += "CXX=#{opt[:cross]}-g++ " env_comp += "LD=#{opt[:cross]}-gcc " @@ -116,14 +127,19 @@ task 'dlls' do end end -desc 'Build the native gem file under rake_compiler_dock' -task 'gem:native' do +desc 'Build the native gem file under rake_compiler_dock. Optionally one can pass argument to build only native gem for a chosen platform.' +task 'gem:native', [:plat] do |t, args| verbose = ENV['V'] || '0' grpc_config = ENV['GRPC_CONFIG'] || 'opt' ruby_cc_versions = ['3.0.0', '2.7.0', '2.6.0', '2.5.0'].join(':') + selected_plat = "#{args[:plat]}" if RUBY_PLATFORM =~ /darwin/ + if !selected_plat.empty? && selected_plat != 'darwin' + fail "Cannot pass platform as an argument when on Darwin." + end + FileUtils.touch 'grpc_c.32.ruby' FileUtils.touch 'grpc_c.64.ruby' unless '2.5' == /(\d+\.\d+)/.match(RUBY_VERSION).to_s @@ -136,8 +152,31 @@ task 'gem:native' do # use env variable to set artifact build paralellism nproc_override = ENV['GRPC_RUBY_BUILD_PROCS'] || `nproc`.strip - Rake::Task['dlls'].execute - ['x86-mingw32', 'x64-mingw32'].each do |plat| + supported_windows_platforms = ['x86-mingw32', 'x64-mingw32'] + supported_unix_platforms = ['x86_64-linux', 'x86-linux', 'x86_64-darwin', 'arm64-darwin'] + supported_platforms = supported_windows_platforms + supported_unix_platforms + + if selected_plat.empty? + # build everything + windows_platforms = supported_windows_platforms + unix_platforms = supported_unix_platforms + else + # build only selected platform + if supported_windows_platforms.include?(selected_plat) + windows_platforms = [selected_plat] + unix_platforms = [] + elsif supported_unix_platforms.include?(selected_plat) + windows_platforms = [] + unix_platforms = [selected_plat] + else + fail "Unsupported platform '#{selected_plat}' passed as an argument." + end + end + + # Create the windows dlls or create the empty placeholders + Rake::Task['dlls'].execute(plat: windows_platforms) + + windows_platforms.each do |plat| run_rake_compiler(plat, <<~EOT) gem update --system --no-document && \ bundle && \ @@ -149,10 +188,13 @@ task 'gem:native' do GRPC_RUBY_BUILD_PROCS=#{nproc_override} EOT end - # Truncate grpc_c.*.ruby files because they're for Windows only. + + # Truncate grpc_c.*.ruby files because they're for Windows only and we don't want + # them to take up space in the gems that don't target windows. File.truncate('grpc_c.32.ruby', 0) File.truncate('grpc_c.64.ruby', 0) - ['x86_64-linux', 'x86-linux', 'x86_64-darwin', 'arm64-darwin'].each do |plat| + + unix_platforms.each do |plat| run_rake_compiler(plat, <<~EOT) gem update --system --no-document && \ bundle && \ diff --git a/tools/internal_ci/helper_scripts/prepare_build_linux_ruby_artifact_rc b/tools/internal_ci/helper_scripts/prepare_build_linux_ruby_artifact_rc new file mode 100644 index 00000000000..fc9910a04c2 --- /dev/null +++ b/tools/internal_ci/helper_scripts/prepare_build_linux_ruby_artifact_rc @@ -0,0 +1,46 @@ +#!/bin/bash +# Copyright 2021 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Source this rc script to load RVM and prepare the ruby environment +# for building ruby artifacts. + +# sourcing rvm can generate a failure and we don't want to quit there +set +ex +# Look for rvm either in /etc/profile.d or in $HOME +# shellcheck disable=SC1091 +[[ -s /etc/profile.d/rvm.sh ]] && source /etc/profile.d/rvm.sh +# shellcheck disable=SC1090 +[[ -s "$HOME/.rvm/scripts/rvm" ]] && source "$HOME/.rvm/scripts/rvm" + +# rvm commands are very verbose and we dont want to pollute the log by echo, +# but we want to exit if there's a failure +set -e + +echo "rvm install ruby-2.5.7" +time rvm install ruby-2.5.7 +echo "rvm --default use ruby-2.5.7" +rvm --default use ruby-2.5.7 + +# restore the original echo and exit on failure behavior +set -ex + +# print current ruby version to log +ruby --version + +# Bundler is required for grpc ruby artifact build. +gem install bundler -v 1.17.3 + +# log gem versions for easier debugging if things go wrong +gem list || true diff --git a/tools/internal_ci/linux/grpc_build_artifacts.sh b/tools/internal_ci/linux/grpc_build_artifacts.sh index 38c39cbf6af..ad08a086977 100755 --- a/tools/internal_ci/linux/grpc_build_artifacts.sh +++ b/tools/internal_ci/linux/grpc_build_artifacts.sh @@ -23,14 +23,10 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_rc -set +ex -[[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh -set -e # rvm commands are very verbose -rvm install ruby-2.5.7 -rvm --default use ruby-2.5.7 -set -ex +# prerequisites for ruby artifact build on linux +source tools/internal_ci/helper_scripts/prepare_build_linux_ruby_artifact_rc -tools/run_tests/task_runner.py -f artifact linux ${TASK_RUNNER_EXTRA_FILTERS} -j 12 || FAILED="true" +tools/run_tests/task_runner.py -f artifact linux ${TASK_RUNNER_EXTRA_FILTERS} -j 6 --inner_jobs 6 || FAILED="true" tools/internal_ci/helper_scripts/store_artifacts_from_moved_src_tree.sh diff --git a/tools/internal_ci/linux/grpc_build_packages.sh b/tools/internal_ci/linux/grpc_build_packages.sh index 4ce6d2e1c05..759660491ef 100644 --- a/tools/internal_ci/linux/grpc_build_packages.sh +++ b/tools/internal_ci/linux/grpc_build_packages.sh @@ -23,13 +23,6 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_rc -set +ex -[[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh -set -e # rvm commands are very verbose -rvm install ruby-2.5.7 -rvm --default use ruby-2.5.7 -set -ex - # Move artifacts generated by the previous step in the build chain to a place # where they can be accessed from within a docker container that builds # the packages diff --git a/tools/internal_ci/linux/grpc_distribtests.sh b/tools/internal_ci/linux/grpc_distribtests.sh index c71d176d470..776e2c8e034 100644 --- a/tools/internal_ci/linux/grpc_distribtests.sh +++ b/tools/internal_ci/linux/grpc_distribtests.sh @@ -28,13 +28,6 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_rc # under qemu emulator. source tools/internal_ci/helper_scripts/prepare_qemu_rc -set +ex -[[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh -set -e # rvm commands are very verbose -rvm install ruby-2.5.7 -rvm --default use ruby-2.5.7 -set -ex - # Move packages generated by the previous step in the build chain to a place # where they can be accessed from within a docker container that run the # distribtests diff --git a/tools/internal_ci/linux/grpc_distribtests_ruby.sh b/tools/internal_ci/linux/grpc_distribtests_ruby.sh index 6b05650b975..be73cdd3071 100755 --- a/tools/internal_ci/linux/grpc_distribtests_ruby.sh +++ b/tools/internal_ci/linux/grpc_distribtests_ruby.sh @@ -23,15 +23,11 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_rc -set +ex -[[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh -set -e # rvm commands are very verbose -rvm install ruby-2.5.7 -rvm --default use ruby-2.5.7 -set -ex +# prerequisites for ruby artifact build on linux +source tools/internal_ci/helper_scripts/prepare_build_linux_ruby_artifact_rc # Build all ruby linux artifacts (this step actually builds all the binary wheels and source archives) -tools/run_tests/task_runner.py -f artifact linux ruby ${TASK_RUNNER_EXTRA_FILTERS} -j 2 --inner_jobs 16 -x build_artifacts/sponge_log.xml || FAILED="true" +tools/run_tests/task_runner.py -f artifact linux ruby ${TASK_RUNNER_EXTRA_FILTERS} -j 6 --inner_jobs 6 -x build_artifacts/sponge_log.xml || FAILED="true" # Ruby "build_package" step is basically just a passthough for the "grpc" gems, so it's enough to just # copy the native gems directly to the "distribtests" step and skip the "build_package" phase entirely. @@ -48,7 +44,7 @@ cp -r artifacts/ruby_native_gem_*/* input_artifacts/ || true # Run all ruby linux distribtests # We run the distribtests even if some of the artifacts have failed to build, since that gives # a better signal about which distribtest are affected by the currently broken artifact builds. -tools/run_tests/task_runner.py -f distribtest linux ruby ${TASK_RUNNER_EXTRA_FILTERS} -j 6 -x distribtests/sponge_log.xml || FAILED="true" +tools/run_tests/task_runner.py -f distribtest linux ruby ${TASK_RUNNER_EXTRA_FILTERS} -j 12 -x distribtests/sponge_log.xml || FAILED="true" tools/internal_ci/helper_scripts/store_artifacts_from_moved_src_tree.sh diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index 05388c11e2d..d86f484eca6 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -198,11 +198,11 @@ class PythonArtifact: class RubyArtifact: """Builds ruby native gem.""" - def __init__(self, platform, arch, presubmit=False): - self.name = 'ruby_native_gem_%s_%s' % (platform, arch) + def __init__(self, platform, gem_platform, presubmit=False): + self.name = 'ruby_native_gem_%s_%s' % (platform, gem_platform) self.platform = platform - self.arch = arch - self.labels = ['artifact', 'ruby', platform, arch] + self.gem_platform = gem_platform + self.labels = ['artifact', 'ruby', platform, gem_platform] if presubmit: self.labels.append('presubmit') @@ -216,11 +216,13 @@ class RubyArtifact: environ['GRPC_RUBY_BUILD_PROCS'] = str(inner_jobs) # Ruby build uses docker internally and docker cannot be nested. # We are using a custom workspace instead. - return create_jobspec( - self.name, ['tools/run_tests/artifacts/build_artifact_ruby.sh'], - use_workspace=True, - timeout_seconds=90 * 60, - environ=environ) + return create_jobspec(self.name, [ + 'tools/run_tests/artifacts/build_artifact_ruby.sh', + self.gem_platform + ], + use_workspace=True, + timeout_seconds=90 * 60, + environ=environ) class CSharpExtArtifact: @@ -457,8 +459,13 @@ def targets(): PythonArtifact('windows', 'x64', 'Python38'), PythonArtifact('windows', 'x64', 'Python39'), PythonArtifact('windows', 'x64', 'Python310', presubmit=True), - RubyArtifact('linux', 'x64', presubmit=True), - RubyArtifact('macos', 'x64', presubmit=True), + RubyArtifact('linux', 'x86-mingw32', presubmit=True), + RubyArtifact('linux', 'x64-mingw32', presubmit=True), + RubyArtifact('linux', 'x86_64-linux', presubmit=True), + RubyArtifact('linux', 'x86-linux', presubmit=True), + RubyArtifact('linux', 'x86_64-darwin', presubmit=True), + RubyArtifact('linux', 'arm64-darwin', presubmit=True), + RubyArtifact('macos', 'darwin', presubmit=True), PHPArtifact('linux', 'x64', presubmit=True), - PHPArtifact('macos', 'x64', presubmit=True) + PHPArtifact('macos', 'x64', presubmit=True), ]) diff --git a/tools/run_tests/artifacts/build_artifact_ruby.sh b/tools/run_tests/artifacts/build_artifact_ruby.sh index ef6603d37e1..396173751c3 100755 --- a/tools/run_tests/artifacts/build_artifact_ruby.sh +++ b/tools/run_tests/artifacts/build_artifact_ruby.sh @@ -14,15 +14,12 @@ # limitations under the License. set -ex +# the platform for which we wanna build the native gem +GEM_PLATFORM="$1" + SYSTEM=$(uname | cut -f 1 -d_) cd "$(dirname "$0")/../../.." -set +ex -# shellcheck disable=SC1091 -[[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh -# shellcheck disable=SC1090 -[[ -s "$HOME/.rvm/scripts/rvm" ]] && source "$HOME/.rvm/scripts/rvm" -set -ex if [ "$SYSTEM" == "MSYS" ] ; then SYSTEM=MINGW32 @@ -36,22 +33,24 @@ if [ "$SYSTEM" == "MINGW32" ] ; then exit 1 fi -set +ex - -# To workaround the problem with bundler 2.1.0 and rubygems-bundler 1.4.5 -# https://github.com/bundler/bundler/issues/7488 -rvm @global -gem uninstall rubygems-bundler +# log ruby version for easier debugging if things go wrong +# we assume that the current ruby version has already been selected +# (e.g. by the top-level CI script or with rvm locally) +ruby --version -rvm use default -gem install bundler -v 1.17.3 +# log gem versions for easier debugging if things go wrong +gem list || true +# avoid polluting the global gem diretory +# by configuring "bundle install" to install all the gems +# into a project-local directory +export BUNDLE_PATH=bundle_local_gems tools/run_tests/helper_scripts/bundle_install_wrapper.sh -set -ex - +# set the dockerhub org under which all the gRPC's ruby-compiler-dock docker images +# are available. export DOCKERHUB_ORGANIZATION=grpctesting -bundle exec rake gem:native +bundle exec rake "gem:native[${GEM_PLATFORM}]" if [ "$SYSTEM" == "Darwin" ] ; then # TODO: consider rewriting this to pass shellcheck diff --git a/tools/run_tests/helper_scripts/bundle_install_wrapper.sh b/tools/run_tests/helper_scripts/bundle_install_wrapper.sh index ab31dd5c800..690239f1e19 100755 --- a/tools/run_tests/helper_scripts/bundle_install_wrapper.sh +++ b/tools/run_tests/helper_scripts/bundle_install_wrapper.sh @@ -26,6 +26,13 @@ if [ "$SYSTEM" == "Darwin" ] ; then # See suggestion in https://github.com/bundler/bundler/issues/3692 BUNDLE_SPECIFIC_PLATFORM=true bundle install else - bundle install + # TODO(jtattermusch): remove the retry hack + # on linux artifact build, multiple instances of "bundle install" run in parallel + # in different workspaces. That should work fine since the workspaces + # are isolated, but causes occasional + # failures (builder/gem bug?). Retrying fixes the issue. + # Note that using bundle install --retry is not enough because + # that only retries downloading, not installation. + bundle install || (sleep 10; bundle install) fi