From 32b087e6741b15d302fc4a4ea0fb205ae1d41954 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 13 Jan 2022 14:58:48 +0100 Subject: [PATCH] Ruby artifact speedup (#28542) * fix ruby native extension build parallelism defaults * allow overriding ruby artifact build parallelism with GRPC_RUBY_BUILD_PROCS * honor --inner_jobs for ruby artifact build * always start building ruby artifacts first * fine tune grpc_distribtests_ruby.sh * address review feedback * strip newline from nproc output --- Rakefile | 15 +++++++++--- src/ruby/ext/grpc/extconf.rb | 2 +- .../src/ruby/ext/grpc/extconf.rb.template | 2 +- .../linux/grpc_distribtests_ruby.sh | 2 +- tools/run_tests/artifacts/artifact_targets.py | 23 +++++++++++++++---- 5 files changed, 33 insertions(+), 11 deletions(-) diff --git a/Rakefile b/Rakefile index ab311966317..54cc694b3ee 100755 --- a/Rakefile +++ b/Rakefile @@ -82,6 +82,8 @@ desc 'Build the Windows gRPC DLLs for Ruby' task 'dlls' do grpc_config = ENV['GRPC_CONFIG'] || 'opt' verbose = ENV['V'] || '0' + # use env variable to set artifact build paralellism + nproc_override = ENV['GRPC_RUBY_BUILD_PROCS'] || `nproc`.strip env = 'CPPFLAGS="-D_WIN32_WINNT=0x600 -DNTDDI_VERSION=0x06000000 -DUNICODE -D_UNICODE -Wno-unused-variable -Wno-unused-result -DCARES_STATICLIB -Wno-error=conversion -Wno-sign-compare -Wno-parentheses -Wno-format -DWIN32_LEAN_AND_MEAN" ' env += 'CFLAGS="-Wno-incompatible-pointer-types" ' @@ -93,6 +95,8 @@ task 'dlls' do env += 'EMBED_CARES=true ' env += 'BUILDDIR=/tmp ' env += "V=#{verbose} " + env += "GRPC_RUBY_BUILD_PROCS=#{nproc_override} " + out = GrpcBuildConfig::CORE_WINDOWS_DLL w64 = { cross: 'x86_64-w64-mingw32', out: 'grpc_c.64.ruby', platform: 'x64-mingw32' } @@ -105,7 +109,7 @@ task 'dlls' do env_comp += "LDXX=#{opt[:cross]}-g++ " run_rake_compiler(opt[:platform], <<~EOT) gem update --system --no-document && \ - #{env} #{env_comp} make -j`nproc` #{out} && \ + #{env} #{env_comp} make -j#{nproc_override} #{out} && \ #{opt[:cross]}-strip -x -S #{out} && \ cp #{out} #{opt[:out]} EOT @@ -129,6 +133,9 @@ task 'gem:native' do end system "bundle exec rake cross native gem RUBY_CC_VERSION=#{ruby_cc_versions} V=#{verbose} GRPC_CONFIG=#{grpc_config}" else + # use env variable to set artifact build paralellism + nproc_override = ENV['GRPC_RUBY_BUILD_PROCS'] || `nproc`.strip + Rake::Task['dlls'].execute ['x86-mingw32', 'x64-mingw32'].each do |plat| run_rake_compiler(plat, <<~EOT) @@ -138,7 +145,8 @@ task 'gem:native' do bundle exec rake native:#{plat} pkg/#{spec.full_name}-#{plat}.gem pkg/#{spec.full_name}.gem \ RUBY_CC_VERSION=#{ruby_cc_versions} \ V=#{verbose} \ - GRPC_CONFIG=#{grpc_config} + GRPC_CONFIG=#{grpc_config} \ + GRPC_RUBY_BUILD_PROCS=#{nproc_override} EOT end # Truncate grpc_c.*.ruby files because they're for Windows only. @@ -152,7 +160,8 @@ task 'gem:native' do bundle exec rake native:#{plat} pkg/#{spec.full_name}-#{plat}.gem pkg/#{spec.full_name}.gem \ RUBY_CC_VERSION=#{ruby_cc_versions} \ V=#{verbose} \ - GRPC_CONFIG=#{grpc_config} + GRPC_CONFIG=#{grpc_config} \ + GRPC_RUBY_BUILD_PROCS=#{nproc_override} EOT end end diff --git a/src/ruby/ext/grpc/extconf.rb b/src/ruby/ext/grpc/extconf.rb index 0b2e2c4bad4..82fd9e34a9a 100644 --- a/src/ruby/ext/grpc/extconf.rb +++ b/src/ruby/ext/grpc/extconf.rb @@ -69,7 +69,7 @@ ENV['BUILDDIR'] = output_dir unless windows puts 'Building internal gRPC into ' + grpc_lib_dir nproc = 4 - nproc = Etc.nprocessors * 2 if Etc.respond_to? :nprocessors + nproc = Etc.nprocessors if Etc.respond_to? :nprocessors nproc_override = ENV['GRPC_RUBY_BUILD_PROCS'] unless nproc_override.nil? or nproc_override.size == 0 nproc = nproc_override diff --git a/templates/src/ruby/ext/grpc/extconf.rb.template b/templates/src/ruby/ext/grpc/extconf.rb.template index 070370abb31..6b6e1343b03 100644 --- a/templates/src/ruby/ext/grpc/extconf.rb.template +++ b/templates/src/ruby/ext/grpc/extconf.rb.template @@ -71,7 +71,7 @@ unless windows puts 'Building internal gRPC into ' + grpc_lib_dir nproc = 4 - nproc = Etc.nprocessors * 2 if Etc.respond_to? :nprocessors + nproc = Etc.nprocessors if Etc.respond_to? :nprocessors nproc_override = ENV['GRPC_RUBY_BUILD_PROCS'] unless nproc_override.nil? or nproc_override.size == 0 nproc = nproc_override diff --git a/tools/internal_ci/linux/grpc_distribtests_ruby.sh b/tools/internal_ci/linux/grpc_distribtests_ruby.sh index 710c14da363..6b05650b975 100755 --- a/tools/internal_ci/linux/grpc_distribtests_ruby.sh +++ b/tools/internal_ci/linux/grpc_distribtests_ruby.sh @@ -31,7 +31,7 @@ rvm --default use ruby-2.5.7 set -ex # Build all ruby linux artifacts (this step actually builds all the binary wheels and source archives) -tools/run_tests/task_runner.py -f artifact linux ruby ${TASK_RUNNER_EXTRA_FILTERS} -j 6 -x build_artifacts/sponge_log.xml || FAILED="true" +tools/run_tests/task_runner.py -f artifact linux ruby ${TASK_RUNNER_EXTRA_FILTERS} -j 2 --inner_jobs 16 -x build_artifacts/sponge_log.xml || FAILED="true" # Ruby "build_package" step is basically just a passthough for the "grpc" gems, so it's enough to just # copy the native gems directly to the "distribtests" step and skip the "build_package" phase entirely. diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index ec1dbcc7c75..05388c11e2d 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -210,14 +210,17 @@ class RubyArtifact: return [] def build_jobspec(self, inner_jobs=None): - # TODO(jtattermusch): honor inner_jobs arg for this task. - del inner_jobs + environ = {} + if inner_jobs is not None: + # set number of parallel jobs when building native extension + environ['GRPC_RUBY_BUILD_PROCS'] = str(inner_jobs) # Ruby build uses docker internally and docker cannot be nested. # We are using a custom workspace instead. return create_jobspec( self.name, ['tools/run_tests/artifacts/build_artifact_ruby.sh'], use_workspace=True, - timeout_seconds=90 * 60) + timeout_seconds=90 * 60, + environ=environ) class CSharpExtArtifact: @@ -376,9 +379,19 @@ class ProtocArtifact: return self.name +def _reorder_targets_for_build_speed(targets): + """Reorder targets to achieve optimal build speed""" + # ruby artifact build builds multiple artifacts at once, so make sure + # we start building ruby artifacts first, so that they don't end up + # being a long tail once everything else finishes. + return list( + sorted(targets, + key=lambda target: 0 if target.name.startswith('ruby_') else 1)) + + def targets(): """Gets list of supported targets""" - return [ + return _reorder_targets_for_build_speed([ ProtocArtifact('linux', 'x64', presubmit=True), ProtocArtifact('linux', 'x86', presubmit=True), ProtocArtifact('linux', 'aarch64', presubmit=True), @@ -448,4 +461,4 @@ def targets(): RubyArtifact('macos', 'x64', presubmit=True), PHPArtifact('linux', 'x64', presubmit=True), PHPArtifact('macos', 'x64', presubmit=True) - ] + ])