[ruby]: add pre-compiled binaries for ruby 3.2; drop them for ruby 2.6 (#32089)

[ruby]: add pre-compiled binaries for ruby 3.2; drop them for ruby 2.6

Rakefile

@@ -144,91 +144,75 @@ task 'gem:native', [:plat] do |t, args|
   verbose = ENV['V'] || '0'
 
   grpc_config = ENV['GRPC_CONFIG'] || 'opt'
-  ruby_cc_versions = ['3.1.0', '3.0.0', '2.7.0', '2.6.0'].join(':')
+  ruby_cc_versions = ['3.2.0', '3.1.0', '3.0.0', '2.7.0'].join(':')
   selected_plat = "#{args[:plat]}"
 
-  if RUBY_PLATFORM =~ /darwin/
+  # use env variable to set artifact build paralellism
-    if !selected_plat.empty? && selected_plat != 'darwin'
+  nproc_override = ENV['GRPC_RUBY_BUILD_PROCS'] || `nproc`.strip
-      fail "Cannot pass platform as an argument when on Darwin."
-    end
 
-    FileUtils.touch 'grpc_c.32-msvcrt.ruby'
+  # propagate env variables with ccache configuration to the rake-compiler-dock docker container
-    FileUtils.touch 'grpc_c.64-msvcrt.ruby'
+  # and setup ccache symlinks as needed.
-    FileUtils.touch 'grpc_c.64-ucrt.ruby'
+  prepare_ccache_cmd = "export GRPC_BUILD_ENABLE_CCACHE=\"#{ENV.fetch('GRPC_BUILD_ENABLE_CCACHE', '')}\" && "
-    unless '2.5' == /(\d+\.\d+)/.match(RUBY_VERSION).to_s
+  prepare_ccache_cmd += "export CCACHE_SECONDARY_STORAGE=\"#{ENV.fetch('CCACHE_SECONDARY_STORAGE', '')}\" && "
-      fail "rake gem:native (the rake task to build the binary packages) is being " \
+  prepare_ccache_cmd += "export PATH=\"$PATH:/usr/local/bin\" && "
-        "invoked on macos with ruby #{RUBY_VERSION}. The ruby macos artifact " \
+  prepare_ccache_cmd += "source tools/internal_ci/helper_scripts/prepare_ccache_symlinks_rc "
-        "build should be running on ruby 2.5."
+
-    end
+  supported_windows_platforms = ['x86-mingw32', 'x64-mingw32', 'x64-mingw-ucrt']
-    system "bundle exec rake cross native gem RUBY_CC_VERSION=#{ruby_cc_versions} V=#{verbose} GRPC_CONFIG=#{grpc_config}"
+  supported_unix_platforms = ['x86_64-linux', 'x86-linux', 'x86_64-darwin', 'arm64-darwin']
+  supported_platforms = supported_windows_platforms + supported_unix_platforms
+
+  if selected_plat.empty?
+    # build everything
+    windows_platforms = supported_windows_platforms
+    unix_platforms = supported_unix_platforms
   else
-    # use env variable to set artifact build paralellism
+    # build only selected platform
-    nproc_override = ENV['GRPC_RUBY_BUILD_PROCS'] || `nproc`.strip
+    if supported_windows_platforms.include?(selected_plat)
-
+      windows_platforms = [selected_plat]
-    # propagate env variables with ccache configuration to the rake-compiler-dock docker container
+      unix_platforms = []
-    # and setup ccache symlinks as needed.
+    elsif supported_unix_platforms.include?(selected_plat)
-    prepare_ccache_cmd = "export GRPC_BUILD_ENABLE_CCACHE=\"#{ENV.fetch('GRPC_BUILD_ENABLE_CCACHE', '')}\" && "
+      windows_platforms = []
-    prepare_ccache_cmd += "export CCACHE_SECONDARY_STORAGE=\"#{ENV.fetch('CCACHE_SECONDARY_STORAGE', '')}\" && "
+      unix_platforms = [selected_plat]
-    prepare_ccache_cmd += "export PATH=\"$PATH:/usr/local/bin\" && "
-    prepare_ccache_cmd += "source tools/internal_ci/helper_scripts/prepare_ccache_symlinks_rc "
-
-    supported_windows_platforms = ['x86-mingw32', 'x64-mingw32', 'x64-mingw-ucrt']
-    supported_unix_platforms = ['x86_64-linux', 'x86-linux', 'x86_64-darwin', 'arm64-darwin']
-    supported_platforms = supported_windows_platforms + supported_unix_platforms
-
-    if selected_plat.empty?
-      # build everything
-      windows_platforms = supported_windows_platforms
-      unix_platforms = supported_unix_platforms
     else
-      # build only selected platform
+      fail "Unsupported platform '#{selected_plat}' passed as an argument."
-      if supported_windows_platforms.include?(selected_plat)
-        windows_platforms = [selected_plat]
-        unix_platforms = []
-      elsif supported_unix_platforms.include?(selected_plat)
-        windows_platforms = []
-        unix_platforms = [selected_plat]
-      else
-        fail "Unsupported platform '#{selected_plat}' passed as an argument."
-      end
     end
+  end
 
-    # Create the windows dlls or create the empty placeholders
+  # Create the windows dlls or create the empty placeholders
-    Rake::Task['dlls'].execute(plat: windows_platforms)
+  Rake::Task['dlls'].execute(plat: windows_platforms)
-
-    windows_platforms.each do |plat|
-      run_rake_compiler(plat, <<~EOT)
-        #{prepare_ccache_cmd} && \
-        gem update --system --no-document && \
-        bundle && \
-        bundle exec rake clean && \
-        bundle exec rake native:#{plat} pkg/#{spec.full_name}-#{plat}.gem pkg/#{spec.full_name}.gem \
-          RUBY_CC_VERSION=#{ruby_cc_versions} \
-          V=#{verbose} \
-          GRPC_CONFIG=#{grpc_config} \
-          GRPC_RUBY_BUILD_PROCS=#{nproc_override}
-      EOT
-    end
 
-    # Truncate grpc_c.*.ruby files because they're for Windows only and we don't want
+  windows_platforms.each do |plat|
-    # them to take up space in the gems that don't target windows.
+    run_rake_compiler(plat, <<~EOT)
-    File.truncate('grpc_c.32-msvcrt.ruby', 0)
+      #{prepare_ccache_cmd} && \
-    File.truncate('grpc_c.64-msvcrt.ruby', 0)
+      gem update --system --no-document && \
-    File.truncate('grpc_c.64-ucrt.ruby', 0)
+      bundle && \
-
+      bundle exec rake clean && \
-    unix_platforms.each do |plat|
+      bundle exec rake native:#{plat} pkg/#{spec.full_name}-#{plat}.gem pkg/#{spec.full_name}.gem \
-      run_rake_compiler(plat, <<~EOT)
+        RUBY_CC_VERSION=#{ruby_cc_versions} \
-        #{prepare_ccache_cmd} && \
+        V=#{verbose} \
-        gem update --system --no-document && \
+        GRPC_CONFIG=#{grpc_config} \
-        bundle && \
+        GRPC_RUBY_BUILD_PROCS=#{nproc_override}
-        bundle exec rake clean && \
+    EOT
-        bundle exec rake native:#{plat} pkg/#{spec.full_name}-#{plat}.gem pkg/#{spec.full_name}.gem \
+  end
-          RUBY_CC_VERSION=#{ruby_cc_versions} \
+
-          V=#{verbose} \
+  # Truncate grpc_c.*.ruby files because they're for Windows only and we don't want
-          GRPC_CONFIG=#{grpc_config} \
+  # them to take up space in the gems that don't target windows.
-          GRPC_RUBY_BUILD_PROCS=#{nproc_override}
+  File.truncate('grpc_c.32-msvcrt.ruby', 0)
-      EOT
+  File.truncate('grpc_c.64-msvcrt.ruby', 0)
-    end
+  File.truncate('grpc_c.64-ucrt.ruby', 0)
+
+  unix_platforms.each do |plat|
+    run_rake_compiler(plat, <<~EOT)
+      #{prepare_ccache_cmd} && \
+      gem update --system --no-document && \
+      bundle && \
+      bundle exec rake clean && \
+      bundle exec rake native:#{plat} pkg/#{spec.full_name}-#{plat}.gem pkg/#{spec.full_name}.gem \
+        RUBY_CC_VERSION=#{ruby_cc_versions} \
+        V=#{verbose} \
+        GRPC_CONFIG=#{grpc_config} \
+        GRPC_RUBY_BUILD_PROCS=#{nproc_override}
+    EOT
   end
 end
 

grpc.gemspec

@@ -38,8 +38,8 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'logging',            '~> 2.0'
   s.add_development_dependency 'simplecov',          '~> 0.22'
   s.add_development_dependency 'rake',               '~> 13.0'
-  s.add_development_dependency 'rake-compiler',      '<= 1.1.1'
+  s.add_development_dependency 'rake-compiler',      '~> 1.2.1'
-  s.add_development_dependency 'rake-compiler-dock', '~> 1.2'
+  s.add_development_dependency 'rake-compiler-dock', '~> 1.3'
   s.add_development_dependency 'rspec',              '~> 3.6'
   s.add_development_dependency 'rubocop',            '~> 1.41.0'
   s.add_development_dependency 'signet',             '~> 0.7'

src/ruby/ext/grpc/extconf.rb

@@ -187,3 +187,22 @@ if grpc_config == 'opt'
   end
   File.rename('Makefile.new', 'Makefile')
 end
+if ENV['GRPC_RUBY_TEST_ONLY_WORKAROUND_MAKE_INSTALL_BUG']
+  # Note: this env var setting is intended to work around a problem observed
+  # with the ginstall command on grpc's macos automated test infrastructure,
+  # and is not  guaranteed to work in the wild.
+  # Also see https://github.com/rake-compiler/rake-compiler/issues/210.
+  puts 'Overriding the generated Makefile install target to use cp'
+  File.open('Makefile.new', 'w') do |o|
+    File.foreach('Makefile') do |i|
+      if i.start_with?('INSTALL_PROG = ')
+        override = 'INSTALL_PROG = cp'
+        puts "Replacing generated Makefile line: |#{i}|, with: |#{override}|"
+        o.puts override
+      else
+        o.puts i
+      end
+    end
+  end
+  File.rename('Makefile.new', 'Makefile')
+end

templates/grpc.gemspec.template

@@ -40,8 +40,8 @@
     s.add_development_dependency 'logging',            '~> 2.0'
     s.add_development_dependency 'simplecov',          '~> 0.22'
     s.add_development_dependency 'rake',               '~> 13.0'
-    s.add_development_dependency 'rake-compiler',      '<= 1.1.1'
+    s.add_development_dependency 'rake-compiler',      '~> 1.2.1'
-    s.add_development_dependency 'rake-compiler-dock', '~> 1.2'
+    s.add_development_dependency 'rake-compiler-dock', '~> 1.3'
     s.add_development_dependency 'rspec',              '~> 3.6'
     s.add_development_dependency 'rubocop',            '~> 1.41.0'
     s.add_development_dependency 'signet',             '~> 0.7'

third_party/rake-compiler-dock/rake_arm64-darwin.current_version

@@ -1 +1 @@
-us-docker.pkg.dev/grpc-testing/testing-images-public/rake_arm64-darwin:69742889e98f7c0dc2d26cc89a43017179515167@sha256:d1b0c35c651a1911ba290184f63c5e81b17fd20291acea20c91e9f2c2d7c8d50
+us-docker.pkg.dev/grpc-testing/testing-images-public/rake_arm64-darwin:ef464134d73d8d261b7edb53b737c2dc797f7cee@sha256:e0eb1f9f632fb18d4f244b7297d1a5e7cf60ae58e649ac5b2f8ac6266ea07128

third_party/rake-compiler-dock/rake_arm64-darwin/Dockerfile

@@ -1 +1 @@
-FROM larskanis/rake-compiler-dock-mri-arm64-darwin:1.2.2
+FROM ghcr.io/rake-compiler/rake-compiler-dock-image:1.3.0-mri-arm64-darwin

third_party/rake-compiler-dock/rake_x64-mingw-ucrt.current_version

@@ -1 +1 @@
-us-docker.pkg.dev/grpc-testing/testing-images-public/rake_x64-mingw-ucrt:0b8bed3e125d8952ccc8938f905d9792c84077f5@sha256:ab967122a47b0385a1e66b5b91693d971468a276000ac1c066c54bef577ef4b7
+us-docker.pkg.dev/grpc-testing/testing-images-public/rake_x64-mingw-ucrt:3c2e3bdaf402bb1dd329309b8682890cd3cca19f@sha256:63490b0000c6011a19983fef637efc69a2ae0f67b7a4e29cd36db53c881e908d

third_party/rake-compiler-dock/rake_x64-mingw-ucrt/Dockerfile

@@ -1,4 +1,4 @@
-FROM larskanis/rake-compiler-dock-mri-x64-mingw-ucrt:1.2.2
+FROM ghcr.io/rake-compiler/rake-compiler-dock-image:1.3.0-mri-x64-mingw-ucrt
 
 RUN find / -name win32.h | while read f ; do sed -i 's/gettimeofday/rb_gettimeofday/' $f ; done
 

third_party/rake-compiler-dock/rake_x64-mingw32.current_version

@@ -1 +1 @@
-us-docker.pkg.dev/grpc-testing/testing-images-public/rake_x64-mingw32:47e598526ac5068d348e29237e1dc84e3d01afab@sha256:6f4386f2954821c199d104e9dfe250ef99fdb74f4840311f8a7812c462629a83
+us-docker.pkg.dev/grpc-testing/testing-images-public/rake_x64-mingw32:f24841cbe57e448e0cc98498a0a7d9467f91627b@sha256:63ece6e9b336b7cbf66eaa0201505b0579ac06cd7802f19b44c3a816d5617c17

third_party/rake-compiler-dock/rake_x64-mingw32/Dockerfile

@@ -1,4 +1,4 @@
-FROM larskanis/rake-compiler-dock-mri-x64-mingw32:1.2.2
+FROM ghcr.io/rake-compiler/rake-compiler-dock-image:1.3.0-mri-x64-mingw32
 
 RUN find / -name win32.h | while read f ; do sed -i 's/gettimeofday/rb_gettimeofday/' $f ; done
 

third_party/rake-compiler-dock/rake_x86-linux.current_version

@@ -1 +1 @@
-us-docker.pkg.dev/grpc-testing/testing-images-public/rake_x86-linux:e8ffbe97d21d829ae38ab10ef65cf38e542ee8b9@sha256:dab334b2c651e78f82c9ff7141699ac23365676f3972cff07382f7146702b88f
+us-docker.pkg.dev/grpc-testing/testing-images-public/rake_x86-linux:921974cb95c4ca19ef79be0745df716a9bbf1211@sha256:71e3afca0843bf7bd5da7fa04bff40ad976e76aa5867936166b30251d0a692d8

third_party/rake-compiler-dock/rake_x86-linux/Dockerfile

@@ -1,4 +1,4 @@
-FROM larskanis/rake-compiler-dock-mri-x86-linux:1.2.2
+FROM ghcr.io/rake-compiler/rake-compiler-dock-image:1.3.0-mri-x86-linux
 
 #=================
 # Install ccache

third_party/rake-compiler-dock/rake_x86-mingw32.current_version

@@ -1 +1 @@
-us-docker.pkg.dev/grpc-testing/testing-images-public/rake_x86-mingw32:91c4f180dce2a3f4fc9bcf66dc0818f0d8c81415@sha256:09b3f47c5eb8f6b686af4d6cba636637ab310ef1dcfa30e05bb00b183b1b619b
+us-docker.pkg.dev/grpc-testing/testing-images-public/rake_x86-mingw32:bf745e7ec6ab838e4f982e25f086d6c8667f71cc@sha256:629be8f57e2d50b123584f2cfa00ff5b968cc2cc3b2a6b874acd07100a3eb96d

third_party/rake-compiler-dock/rake_x86-mingw32/Dockerfile

@@ -1,4 +1,4 @@
-FROM larskanis/rake-compiler-dock-mri-x86-mingw32:1.2.2
+FROM ghcr.io/rake-compiler/rake-compiler-dock-image:1.3.0-mri-x86-mingw32
 
 RUN find / -name win32.h | while read f ; do sed -i 's/gettimeofday/rb_gettimeofday/' $f ; done
 

third_party/rake-compiler-dock/rake_x86_64-darwin.current_version

@@ -1 +1 @@
-us-docker.pkg.dev/grpc-testing/testing-images-public/rake_x86_64-darwin:56f7dd2c720f4c10a6d4be6a869b11ec1e39a0f3@sha256:b086e560ef215180b843d1d7fe48e2cfbed3112ec4ab57e655ad5dfb61090726
+us-docker.pkg.dev/grpc-testing/testing-images-public/rake_x86_64-darwin:03fc718cb9cc9a32b16f1ed270b12ba77b862a05@sha256:8dd11cad778d9fc01c3555a57254016f5db7227309d24f50a192a6db80d4a51c

third_party/rake-compiler-dock/rake_x86_64-darwin/Dockerfile

@@ -1 +1 @@
-FROM larskanis/rake-compiler-dock-mri-x86_64-darwin:1.2.2
+FROM ghcr.io/rake-compiler/rake-compiler-dock-image:1.3.0-mri-x86_64-darwin

third_party/rake-compiler-dock/rake_x86_64-linux.current_version

@@ -1 +1 @@
-us-docker.pkg.dev/grpc-testing/testing-images-public/rake_x86_64-linux:47c34d44a509c824ae84aaff82c14de5aa3bdc12@sha256:5c485b0a0e6bf5e0fc87122b3d49003a0d0aa30cbaca7250902f53823e50bc9c
+us-docker.pkg.dev/grpc-testing/testing-images-public/rake_x86_64-linux:73e8372e1b5c1c414a5a8c6b37e76ed0d60a26cc@sha256:9aa77587fa4d4c25c91d0ccca8eb806cf0738a4b67eb4b54d40324185658194e

third_party/rake-compiler-dock/rake_x86_64-linux/Dockerfile

@@ -1,4 +1,4 @@
-FROM larskanis/rake-compiler-dock-mri-x86_64-linux:1.2.2
+FROM ghcr.io/rake-compiler/rake-compiler-dock-image:1.3.0-mri-x86_64-linux
 
 #=================
 # Install ccache

tools/dockerfile/distribtest/ruby_stretch_x64_ruby_2_6.current_version

@@ -1 +0,0 @@
-us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_stretch_x64_ruby_2_6:4b215aa6d7029aaa63679881d54b8ca620e2d775@sha256:b9cc471908386767a9dbf3baffb7a67aa93c4cd1057c4ba8572c691c8ba4b3e6

tools/dockerfile/distribtest/ruby_stretch_x64_ruby_2_6/Dockerfile

@@ -1,42 +0,0 @@
-# Copyright 2015 gRPC authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-FROM debian:stretch
-
-# Install Git and basic packages.
-RUN apt-get update && apt-get install -y \
-  gnupg2 \
-  procps \
-  curl \
-  gcc && apt-get clean
-
-#==================
-# Ruby dependencies
-
-# Install rvm
-RUN gpg2 --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
-RUN \curl -sSL https://get.rvm.io | bash -s stable
-
-# Install Ruby 2.6
-RUN /bin/bash -l -c "rvm install ruby-2.6.5"
-RUN /bin/bash -l -c "rvm use --default ruby-2.6.5"
-RUN /bin/bash -l -c "echo 'gem: --no-document' > ~/.gemrc"
-RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc"
-RUN /bin/bash -l -c "echo 'rvm --default use ruby-2.6.5' >> ~/.bashrc"
-RUN /bin/bash -l -c "gem install bundler --no-document"
-
-RUN mkdir /var/local/jenkins
-
-# Define the default command.
-CMD ["bash"]

tools/run_tests/artifacts/distribtest_targets.py

@@ -431,12 +431,6 @@ def targets():
         PythonDistribTest('linux', 'x64', 'arch', source=True),
         PythonDistribTest('linux', 'x64', 'ubuntu2004', source=True),
         # Ruby
-        RubyDistribTest('linux',
-                        'x64',
-                        'stretch',
-                        ruby_version='ruby_2_6',
-                        source=True,
-                        presubmit=True),
         RubyDistribTest('linux',
                         'x64',
                         'stretch',

tools/run_tests/helper_scripts/build_ruby.sh

@@ -28,6 +28,12 @@ fi
 cd "$(dirname "$0")/../../.."
 
 rm -rf ./tmp
+
+SYSTEM=$(uname | cut -f 1 -d_)
+if [ "$SYSTEM" == "Darwin" ]; then
+  # work around https://github.com/rake-compiler/rake-compiler/issues/210
+  export GRPC_RUBY_TEST_ONLY_WORKAROUND_MAKE_INSTALL_BUG=true
+fi
 bundle exec rake compile
 
 # build grpc_ruby_plugin