Build ruby artifacts in parallel (#28243)

* build ruby artifacts in parallel

* fine tune grpc_distribtests_ruby.sh parallelism

* fine tune linux/grpc_build_artifacts.sh parallelism

* cleanup in bundle_install_wrapper

* address review comments

Rakefile

@@ -78,12 +78,26 @@ namespace :suite do
   end
 end
 
-desc 'Build the Windows gRPC DLLs for Ruby'
+desc 'Build the Windows gRPC DLLs for Ruby. The argument contains the list of platforms for which to build dll. Empty placeholder files will be created for platforms that were not selected.'
-task 'dlls' do
+task 'dlls', [:plat] do |t, args|
   grpc_config = ENV['GRPC_CONFIG'] || 'opt'
   verbose = ENV['V'] || '0'
   # use env variable to set artifact build paralellism
   nproc_override = ENV['GRPC_RUBY_BUILD_PROCS'] || `nproc`.strip
+  plat_list = args[:plat]
+
+  build_configs = []
+  w64 = { cross: 'x86_64-w64-mingw32', out: 'grpc_c.64.ruby', platform: 'x64-mingw32' }
+  w32 = { cross: 'i686-w64-mingw32', out: 'grpc_c.32.ruby', platform: 'x86-mingw32' }
+  [w64, w32].each do |config|
+    if plat_list.include?(config[:platform])
+      # build the DLL (as grpc_c.*.ruby)
+      build_configs.append(config)
+    else
+      # create an empty grpc_c.*.ruby file as a placeholder
+      FileUtils.touch config[:out]
+    end
+  end
 
   env = 'CPPFLAGS="-D_WIN32_WINNT=0x600 -DNTDDI_VERSION=0x06000000 -DUNICODE -D_UNICODE -Wno-unused-variable -Wno-unused-result -DCARES_STATICLIB -Wno-error=conversion -Wno-sign-compare -Wno-parentheses -Wno-format -DWIN32_LEAN_AND_MEAN" '
   env += 'CFLAGS="-Wno-incompatible-pointer-types" '
@@ -99,10 +113,7 @@ task 'dlls' do
 
   out = GrpcBuildConfig::CORE_WINDOWS_DLL
 
-  w64 = { cross: 'x86_64-w64-mingw32', out: 'grpc_c.64.ruby', platform: 'x64-mingw32' }
+  build_configs.each do |opt|
-  w32 = { cross: 'i686-w64-mingw32', out: 'grpc_c.32.ruby', platform: 'x86-mingw32' }
-
-  [ w64, w32 ].each do |opt|
     env_comp = "CC=#{opt[:cross]}-gcc "
     env_comp += "CXX=#{opt[:cross]}-g++ "
     env_comp += "LD=#{opt[:cross]}-gcc "
@@ -116,14 +127,19 @@ task 'dlls' do
   end
 end
 
-desc 'Build the native gem file under rake_compiler_dock'
+desc 'Build the native gem file under rake_compiler_dock. Optionally one can pass argument to build only native gem for a chosen platform.'
-task 'gem:native' do
+task 'gem:native', [:plat] do |t, args|
   verbose = ENV['V'] || '0'
 
   grpc_config = ENV['GRPC_CONFIG'] || 'opt'
   ruby_cc_versions = ['3.0.0', '2.7.0', '2.6.0', '2.5.0'].join(':')
+  selected_plat = "#{args[:plat]}"
 
   if RUBY_PLATFORM =~ /darwin/
+    if !selected_plat.empty? && selected_plat != 'darwin'
+      fail "Cannot pass platform as an argument when on Darwin."
+    end
+
     FileUtils.touch 'grpc_c.32.ruby'
     FileUtils.touch 'grpc_c.64.ruby'
     unless '2.5' == /(\d+\.\d+)/.match(RUBY_VERSION).to_s
@@ -136,8 +152,31 @@ task 'gem:native' do
     # use env variable to set artifact build paralellism
     nproc_override = ENV['GRPC_RUBY_BUILD_PROCS'] || `nproc`.strip
 
-    Rake::Task['dlls'].execute
+    supported_windows_platforms = ['x86-mingw32', 'x64-mingw32']
-    ['x86-mingw32', 'x64-mingw32'].each do |plat|
+    supported_unix_platforms = ['x86_64-linux', 'x86-linux', 'x86_64-darwin', 'arm64-darwin']
+    supported_platforms = supported_windows_platforms + supported_unix_platforms
+
+    if selected_plat.empty?
+      # build everything
+      windows_platforms = supported_windows_platforms
+      unix_platforms = supported_unix_platforms
+    else
+      # build only selected platform
+      if supported_windows_platforms.include?(selected_plat)
+        windows_platforms = [selected_plat]
+        unix_platforms = []
+      elsif supported_unix_platforms.include?(selected_plat)
+        windows_platforms = []
+        unix_platforms = [selected_plat]
+      else
+        fail "Unsupported platform '#{selected_plat}' passed as an argument."
+      end
+    end
+
+    # Create the windows dlls or create the empty placeholders
+    Rake::Task['dlls'].execute(plat: windows_platforms)
+
+    windows_platforms.each do |plat|
       run_rake_compiler(plat, <<~EOT)
         gem update --system --no-document && \
         bundle && \
@@ -149,10 +188,13 @@ task 'gem:native' do
           GRPC_RUBY_BUILD_PROCS=#{nproc_override}
       EOT
     end
-    # Truncate grpc_c.*.ruby files because they're for Windows only.
+
+    # Truncate grpc_c.*.ruby files because they're for Windows only and we don't want
+    # them to take up space in the gems that don't target windows.
     File.truncate('grpc_c.32.ruby', 0)
     File.truncate('grpc_c.64.ruby', 0)
-    ['x86_64-linux', 'x86-linux', 'x86_64-darwin', 'arm64-darwin'].each do |plat|
+
+    unix_platforms.each do |plat|
       run_rake_compiler(plat, <<~EOT)
         gem update --system --no-document && \
         bundle && \

tools/internal_ci/helper_scripts/prepare_build_linux_ruby_artifact_rc

@@ -0,0 +1,46 @@
+#!/bin/bash
+# Copyright 2021 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Source this rc script to load RVM and prepare the ruby environment
+# for building ruby artifacts.
+
+# sourcing rvm can generate a failure and we don't want to quit there
+set +ex
+# Look for rvm either in /etc/profile.d or in $HOME
+# shellcheck disable=SC1091
+[[ -s /etc/profile.d/rvm.sh ]] && source /etc/profile.d/rvm.sh
+# shellcheck disable=SC1090
+[[ -s "$HOME/.rvm/scripts/rvm" ]] && source "$HOME/.rvm/scripts/rvm"
+
+# rvm commands are very verbose and we dont want to pollute the log by echo,
+# but we want to exit if there's a failure
+set -e
+
+echo "rvm install ruby-2.5.7"
+time rvm install ruby-2.5.7
+echo "rvm --default use ruby-2.5.7"
+rvm --default use ruby-2.5.7
+
+# restore the original echo and exit on failure behavior
+set -ex
+
+# print current ruby version to log
+ruby --version
+
+# Bundler is required for grpc ruby artifact build.
+gem install bundler -v 1.17.3
+
+# log gem versions for easier debugging if things go wrong
+gem list || true

tools/internal_ci/linux/grpc_build_artifacts.sh

@@ -23,14 +23,10 @@ cd $(dirname $0)/../../..
 
 source tools/internal_ci/helper_scripts/prepare_build_linux_rc
 
-set +ex
+# prerequisites for ruby artifact build on linux
-[[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh
+source tools/internal_ci/helper_scripts/prepare_build_linux_ruby_artifact_rc
-set -e  # rvm commands are very verbose
-rvm install ruby-2.5.7
-rvm --default use ruby-2.5.7
-set -ex
 
-tools/run_tests/task_runner.py -f artifact linux ${TASK_RUNNER_EXTRA_FILTERS} -j 12 || FAILED="true"
+tools/run_tests/task_runner.py -f artifact linux ${TASK_RUNNER_EXTRA_FILTERS} -j 6 --inner_jobs 6 || FAILED="true"
 
 tools/internal_ci/helper_scripts/store_artifacts_from_moved_src_tree.sh
 

tools/internal_ci/linux/grpc_build_packages.sh

@@ -23,13 +23,6 @@ cd $(dirname $0)/../../..
 
 source tools/internal_ci/helper_scripts/prepare_build_linux_rc
 
-set +ex
-[[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh
-set -e  # rvm commands are very verbose
-rvm install ruby-2.5.7
-rvm --default use ruby-2.5.7
-set -ex
-
 # Move artifacts generated by the previous step in the build chain to a place
 # where they can be accessed from within a docker container that builds
 # the packages

tools/internal_ci/linux/grpc_distribtests.sh

@@ -28,13 +28,6 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_rc
 # under qemu emulator.
 source tools/internal_ci/helper_scripts/prepare_qemu_rc
 
-set +ex
-[[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh
-set -e  # rvm commands are very verbose
-rvm install ruby-2.5.7
-rvm --default use ruby-2.5.7
-set -ex
-
 # Move packages generated by the previous step in the build chain to a place
 # where they can be accessed from within a docker container that run the
 # distribtests

tools/internal_ci/linux/grpc_distribtests_ruby.sh

@@ -23,15 +23,11 @@ cd $(dirname $0)/../../..
 
 source tools/internal_ci/helper_scripts/prepare_build_linux_rc
 
-set +ex
+# prerequisites for ruby artifact build on linux
-[[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh
+source tools/internal_ci/helper_scripts/prepare_build_linux_ruby_artifact_rc
-set -e  # rvm commands are very verbose
-rvm install ruby-2.5.7
-rvm --default use ruby-2.5.7
-set -ex
 
 # Build all ruby linux artifacts (this step actually builds all the binary wheels and source archives)
-tools/run_tests/task_runner.py -f artifact linux ruby ${TASK_RUNNER_EXTRA_FILTERS} -j 2 --inner_jobs 16 -x build_artifacts/sponge_log.xml || FAILED="true"
+tools/run_tests/task_runner.py -f artifact linux ruby ${TASK_RUNNER_EXTRA_FILTERS} -j 6 --inner_jobs 6 -x build_artifacts/sponge_log.xml || FAILED="true"
 
 # Ruby "build_package" step is basically just a passthough for the "grpc" gems, so it's enough to just
 # copy the native gems directly to the "distribtests" step and skip the "build_package" phase entirely.
@@ -48,7 +44,7 @@ cp -r artifacts/ruby_native_gem_*/* input_artifacts/ || true
 # Run all ruby linux distribtests
 # We run the distribtests even if some of the artifacts have failed to build, since that gives
 # a better signal about which distribtest are affected by the currently broken artifact builds.
-tools/run_tests/task_runner.py -f distribtest linux ruby ${TASK_RUNNER_EXTRA_FILTERS} -j 6 -x distribtests/sponge_log.xml || FAILED="true"
+tools/run_tests/task_runner.py -f distribtest linux ruby ${TASK_RUNNER_EXTRA_FILTERS} -j 12 -x distribtests/sponge_log.xml || FAILED="true"
 
 tools/internal_ci/helper_scripts/store_artifacts_from_moved_src_tree.sh
 

tools/run_tests/artifacts/artifact_targets.py

@@ -198,11 +198,11 @@ class PythonArtifact:
 class RubyArtifact:
     """Builds ruby native gem."""
 
-    def __init__(self, platform, arch, presubmit=False):
+    def __init__(self, platform, gem_platform, presubmit=False):
-        self.name = 'ruby_native_gem_%s_%s' % (platform, arch)
+        self.name = 'ruby_native_gem_%s_%s' % (platform, gem_platform)
         self.platform = platform
-        self.arch = arch
+        self.gem_platform = gem_platform
-        self.labels = ['artifact', 'ruby', platform, arch]
+        self.labels = ['artifact', 'ruby', platform, gem_platform]
         if presubmit:
             self.labels.append('presubmit')
 
@@ -216,8 +216,10 @@ class RubyArtifact:
             environ['GRPC_RUBY_BUILD_PROCS'] = str(inner_jobs)
         # Ruby build uses docker internally and docker cannot be nested.
         # We are using a custom workspace instead.
-        return create_jobspec(
+        return create_jobspec(self.name, [
-            self.name, ['tools/run_tests/artifacts/build_artifact_ruby.sh'],
+            'tools/run_tests/artifacts/build_artifact_ruby.sh',
+            self.gem_platform
+        ],
                               use_workspace=True,
                               timeout_seconds=90 * 60,
                               environ=environ)
@@ -457,8 +459,13 @@ def targets():
         PythonArtifact('windows', 'x64', 'Python38'),
         PythonArtifact('windows', 'x64', 'Python39'),
         PythonArtifact('windows', 'x64', 'Python310', presubmit=True),
-        RubyArtifact('linux', 'x64', presubmit=True),
+        RubyArtifact('linux', 'x86-mingw32', presubmit=True),
-        RubyArtifact('macos', 'x64', presubmit=True),
+        RubyArtifact('linux', 'x64-mingw32', presubmit=True),
+        RubyArtifact('linux', 'x86_64-linux', presubmit=True),
+        RubyArtifact('linux', 'x86-linux', presubmit=True),
+        RubyArtifact('linux', 'x86_64-darwin', presubmit=True),
+        RubyArtifact('linux', 'arm64-darwin', presubmit=True),
+        RubyArtifact('macos', 'darwin', presubmit=True),
         PHPArtifact('linux', 'x64', presubmit=True),
-        PHPArtifact('macos', 'x64', presubmit=True)
+        PHPArtifact('macos', 'x64', presubmit=True),
     ])

tools/run_tests/artifacts/build_artifact_ruby.sh

@@ -14,15 +14,12 @@
 # limitations under the License.
 set -ex
 
+# the platform for which we wanna build the native gem
+GEM_PLATFORM="$1"
+
 SYSTEM=$(uname | cut -f 1 -d_)
 
 cd "$(dirname "$0")/../../.."
-set +ex
-# shellcheck disable=SC1091
-[[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh
-# shellcheck disable=SC1090
-[[ -s "$HOME/.rvm/scripts/rvm" ]] && source "$HOME/.rvm/scripts/rvm"
-set -ex
 
 if [ "$SYSTEM" == "MSYS" ] ; then
   SYSTEM=MINGW32
@@ -36,22 +33,24 @@ if [ "$SYSTEM" == "MINGW32" ] ; then
   exit 1
 fi
 
-set +ex
+# log ruby version for easier debugging if things go wrong
-
+# we assume that the current ruby version has already been selected
-# To workaround the problem with bundler 2.1.0 and rubygems-bundler 1.4.5
+# (e.g. by the top-level CI script or with rvm locally)
-# https://github.com/bundler/bundler/issues/7488
+ruby --version
-rvm @global
-gem uninstall rubygems-bundler
 
-rvm use default
+# log gem versions for easier debugging if things go wrong
-gem install bundler -v 1.17.3
+gem list || true
 
+# avoid polluting the global gem diretory
+# by configuring "bundle install" to install all the gems
+# into a project-local directory
+export BUNDLE_PATH=bundle_local_gems
 tools/run_tests/helper_scripts/bundle_install_wrapper.sh
 
-set -ex
+# set the dockerhub org under which all the gRPC's ruby-compiler-dock docker images
-
+# are available.
 export DOCKERHUB_ORGANIZATION=grpctesting
-bundle exec rake gem:native
+bundle exec rake "gem:native[${GEM_PLATFORM}]"
 
 if [ "$SYSTEM" == "Darwin" ] ; then
   # TODO: consider rewriting this to pass shellcheck

tools/run_tests/helper_scripts/bundle_install_wrapper.sh

@@ -26,6 +26,13 @@ if [ "$SYSTEM" == "Darwin" ] ; then
   # See suggestion in https://github.com/bundler/bundler/issues/3692
   BUNDLE_SPECIFIC_PLATFORM=true bundle install
 else
-  bundle install
+  # TODO(jtattermusch): remove the retry hack
+  # on linux artifact build, multiple instances of "bundle install" run in parallel
+  # in different workspaces. That should work fine since the workspaces
+  # are isolated, but causes occasional
+  # failures (builder/gem bug?). Retrying fixes the issue.
+  # Note that using bundle install --retry is not enough because
+  # that only retries downloading, not installation.
+  bundle install || (sleep 10; bundle install)
 fi