ruby: build native Darwin gems using rake-compiler-dock (#25794)

* ruby: use squiggly heredoc for rake-compiler-dock commands

* ruby: use `bundler exec` when building native gems

* ruby: clean .{bundle,so} from src/ruby/lib when building native gems

Failing to remove these files between native builds leads
rake-compiler to establish circular dependencies (which may be a bug
in rake-compiler, but this feels like an easy and good thing to do,
anyway).

* ruby: extract linux and darwin RUBY_PLATFORM checks into variables

There were already "windows" and "bsd" flags, so let's improve
consistency and readability, and set a clear pattern for subsequent
flags.

* ruby: rely on rake-compiler-dock v1.1.0 to set no_native correctly

As of v1.1.0 there's no need to set this explicitly anymore; it will
be true whenever the extension is being built in a RCD container.

See https://github.com/rake-compiler/rake-compiler-dock/commit/362890d

* ruby: add "x86_64-darwin" platform gem

Using RCD for this platform unifies the Darwin native gem build
process with the Linux native gems, which should help avoid
inconsistencies in packaging that result in issues like the missing
Ruby 3.0 binaries in #25060.

Please note that this change leaves the "universal-darwin" platform
native gem untouched, but provides a path forward if the project ever
decides to drop "universal" binary support.

Related to:

- #25429
- #25756

Rakefile

@@ -23,12 +23,6 @@ end
 
 # Add the extension compiler task
 Rake::ExtensionTask.new('grpc_c', spec) do |ext|
-  unless RUBY_PLATFORM =~ /darwin/
-    # TODO: also set "no_native to true" for mac if possible. As is,
-    # "no_native" can only be set if the RUBY_PLATFORM doing
-    # cross-compilation is contained in the "ext.cross_platform" array.
-    ext.no_native = true
-  end
   ext.source_pattern = '**/*.{c,h}'
   ext.ext_dir = File.join('src', 'ruby', 'ext', 'grpc')
   ext.lib_dir = File.join('src', 'ruby', 'lib', 'grpc')
@@ -36,6 +30,7 @@ Rake::ExtensionTask.new('grpc_c', spec) do |ext|
   ext.cross_platform = [
     'x86-mingw32', 'x64-mingw32',
     'x86_64-linux', 'x86-linux',
+    'x86_64-darwin',
     'universal-darwin'
   ]
   ext.cross_compiling do |spec|
@@ -47,6 +42,8 @@ Rake::ExtensionTask.new('grpc_c', spec) do |ext|
   end
 end
 
+CLEAN.add "src/ruby/lib/grpc/[0-9].[0-9]", "src/ruby/lib/grpc/grpc_c.{bundle,so}"
+
 # Define the test suites
 SPEC_SUITES = [
   { id: :wrapper, title: 'wrapper layer', files: %w(src/ruby/spec/*.rb) },
@@ -106,14 +103,13 @@ task 'dlls' do
     env_comp += "CXX=#{opt[:cross]}-g++ "
     env_comp += "LD=#{opt[:cross]}-gcc "
     env_comp += "LDXX=#{opt[:cross]}-g++ "
-    run_rake_compiler opt[:platform], <<-EOT
+    run_rake_compiler(opt[:platform], <<~EOT)
       gem update --system --no-document && \
       #{env} #{env_comp} make -j`nproc` #{out} && \
       #{opt[:cross]}-strip -x -S #{out} && \
       cp #{out} #{opt[:out]}
     EOT
   end
-
 end
 
 desc 'Build the native gem file under rake_compiler_dock'
@@ -135,10 +131,11 @@ task 'gem:native' do
   else
     Rake::Task['dlls'].execute
     ['x86-mingw32', 'x64-mingw32'].each do |plat|
-      run_rake_compiler plat, <<-EOT
+      run_rake_compiler(plat, <<~EOT)
         gem update --system --no-document && \
         bundle && \
-        rake native:#{plat} pkg/#{spec.full_name}-#{plat}.gem pkg/#{spec.full_name}.gem \
+        bundle exec rake clean && \
+        bundle exec rake native:#{plat} pkg/#{spec.full_name}-#{plat}.gem pkg/#{spec.full_name}.gem \
           RUBY_CC_VERSION=#{ruby_cc_versions} \
           V=#{verbose} \
           GRPC_CONFIG=#{grpc_config}
@@ -147,11 +144,12 @@ task 'gem:native' do
     # Truncate grpc_c.*.ruby files because they're for Windows only.
     File.truncate('grpc_c.32.ruby', 0)
     File.truncate('grpc_c.64.ruby', 0)
-    ['x86_64-linux', 'x86-linux'].each do |plat|
+    ['x86_64-linux', 'x86-linux', 'x86_64-darwin'].each do |plat|
-      run_rake_compiler plat, <<-EOT
+      run_rake_compiler(plat, <<~EOT)
         gem update --system --no-document && \
         bundle && \
-        rake native:#{plat} pkg/#{spec.full_name}-#{plat}.gem pkg/#{spec.full_name}.gem \
+        bundle exec rake clean && \
+        bundle exec rake native:#{plat} pkg/#{spec.full_name}-#{plat}.gem pkg/#{spec.full_name}.gem \
           RUBY_CC_VERSION=#{ruby_cc_versions} \
           V=#{verbose} \
           GRPC_CONFIG=#{grpc_config}

grpc.gemspec

@@ -41,7 +41,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'simplecov',          '~> 0.14.1'
   s.add_development_dependency 'rake',               '~> 13.0'
   s.add_development_dependency 'rake-compiler',      '~> 1.1'
-  s.add_development_dependency 'rake-compiler-dock', '~> 1.0'
+  s.add_development_dependency 'rake-compiler-dock', '~> 1.1'
   s.add_development_dependency 'rspec',              '~> 3.6'
   s.add_development_dependency 'rubocop',            '~> 0.49.1'
   s.add_development_dependency 'signet',             '~> 0.7'

src/ruby/ext/grpc/extconf.rb

@@ -17,6 +17,9 @@ require 'mkmf'
 
 windows = RUBY_PLATFORM =~ /mingw|mswin/
 bsd = RUBY_PLATFORM =~ /bsd/
+darwin = RUBY_PLATFORM =~ /darwin/
+linux = RUBY_PLATFORM =~ /linux/
+cross_compiling = ENV['RCD_HOST_RUBY_VERSION'] # set by rake-compiler-dock in build containers
 
 grpc_root = File.expand_path(File.join(File.dirname(__FILE__), '../../../..'))
 
@@ -37,17 +40,17 @@ if ENV['LD'].nil? || ENV['LD'].size == 0
     ENV['LD'] = ENV['CC']
 end
 
-if RUBY_PLATFORM =~ /darwin/
+if darwin && !cross_compiling
   ENV['AR'] = 'libtool'
   ENV['ARFLAGS'] = '-o'
- end
+end
 
 ENV['EMBED_OPENSSL'] = 'true'
 ENV['EMBED_ZLIB'] = 'true'
 ENV['EMBED_CARES'] = 'true'
 
 ENV['ARCH_FLAGS'] = RbConfig::CONFIG['ARCH_FLAG']
-if RUBY_PLATFORM =~ /darwin/
+if darwin && !cross_compiling
   if RUBY_PLATFORM =~ /arm64/
     ENV['ARCH_FLAGS'] = '-arch arm64'
   else
@@ -75,8 +78,8 @@ end
 $CFLAGS << ' -I' + File.join(grpc_root, 'include')
 
 ext_export_file = File.join(grpc_root, 'src', 'ruby', 'ext', 'grpc', 'ext-export')
-$LDFLAGS << ' -Wl,--version-script="' + ext_export_file + '.gcc"' if RUBY_PLATFORM =~ /linux/
+$LDFLAGS << ' -Wl,--version-script="' + ext_export_file + '.gcc"' if linux
-$LDFLAGS << ' -Wl,-exported_symbols_list,"' + ext_export_file + '.clang"' if RUBY_PLATFORM =~ /darwin/
+$LDFLAGS << ' -Wl,-exported_symbols_list,"' + ext_export_file + '.clang"' if darwin
 
 $LDFLAGS << ' ' + File.join(grpc_lib_dir, 'libgrpc.a') unless windows
 if grpc_config == 'gcov'
@@ -88,8 +91,8 @@ if grpc_config == 'dbg'
   $CFLAGS << ' -O0 -ggdb3'
 end
 
-$LDFLAGS << ' -Wl,-wrap,memcpy' if RUBY_PLATFORM =~ /linux/
+$LDFLAGS << ' -Wl,-wrap,memcpy' if linux
-$LDFLAGS << ' -static-libgcc -static-libstdc++' if RUBY_PLATFORM =~ /linux/
+$LDFLAGS << ' -static-libgcc -static-libstdc++' if linux
 $LDFLAGS << ' -static' if windows
 
 $CFLAGS << ' -std=c99 '
@@ -102,7 +105,7 @@ puts 'Generating Makefile for ' + output
 create_makefile(output)
 
 strip_tool = RbConfig::CONFIG['STRIP']
-strip_tool = 'strip -x' if RUBY_PLATFORM =~ /darwin/
+strip_tool += ' -x' if darwin
 
 if grpc_config == 'opt'
   File.open('Makefile.new', 'w') do |o|

templates/grpc.gemspec.template

@@ -43,7 +43,7 @@
     s.add_development_dependency 'simplecov',          '~> 0.14.1'
     s.add_development_dependency 'rake',               '~> 13.0'
     s.add_development_dependency 'rake-compiler',      '~> 1.1'
-    s.add_development_dependency 'rake-compiler-dock', '~> 1.0'
+    s.add_development_dependency 'rake-compiler-dock', '~> 1.1'
     s.add_development_dependency 'rspec',              '~> 3.6'
     s.add_development_dependency 'rubocop',            '~> 0.49.1'
     s.add_development_dependency 'signet',             '~> 0.7'

templates/src/ruby/ext/grpc/extconf.rb.template

@@ -19,6 +19,9 @@
   
   windows = RUBY_PLATFORM =~ /mingw|mswin/
   bsd = RUBY_PLATFORM =~ /bsd/
+  darwin = RUBY_PLATFORM =~ /darwin/
+  linux = RUBY_PLATFORM =~ /linux/
+  cross_compiling = ENV['RCD_HOST_RUBY_VERSION'] # set by rake-compiler-dock in build containers
   
   grpc_root = File.expand_path(File.join(File.dirname(__FILE__), '../../../..'))
   
@@ -39,17 +42,17 @@
       ENV['LD'] = ENV['CC']
   end
   
-  if RUBY_PLATFORM =~ /darwin/
+  if darwin && !cross_compiling
     ENV['AR'] = 'libtool'
     ENV['ARFLAGS'] = '-o'
-   end
+  end
   
   ENV['EMBED_OPENSSL'] = 'true'
   ENV['EMBED_ZLIB'] = 'true'
   ENV['EMBED_CARES'] = 'true'
   
   ENV['ARCH_FLAGS'] = RbConfig::CONFIG['ARCH_FLAG']
-  if RUBY_PLATFORM =~ /darwin/
+  if darwin && !cross_compiling
     if RUBY_PLATFORM =~ /arm64/
       ENV['ARCH_FLAGS'] = '-arch arm64'
     else
@@ -77,8 +80,8 @@
   $CFLAGS << ' -I' + File.join(grpc_root, 'include')
   
   ext_export_file = File.join(grpc_root, 'src', 'ruby', 'ext', 'grpc', 'ext-export')
-  $LDFLAGS << ' -Wl,--version-script="' + ext_export_file + '.gcc"' if RUBY_PLATFORM =~ /linux/
+  $LDFLAGS << ' -Wl,--version-script="' + ext_export_file + '.gcc"' if linux
-  $LDFLAGS << ' -Wl,-exported_symbols_list,"' + ext_export_file + '.clang"' if RUBY_PLATFORM =~ /darwin/
+  $LDFLAGS << ' -Wl,-exported_symbols_list,"' + ext_export_file + '.clang"' if darwin
   
   $LDFLAGS << ' ' + File.join(grpc_lib_dir, 'libgrpc.a') unless windows
   if grpc_config == 'gcov'
@@ -90,8 +93,8 @@
     $CFLAGS << ' -O0 -ggdb3'
   end
   
-  $LDFLAGS << ' -Wl,-wrap,memcpy' if RUBY_PLATFORM =~ /linux/
+  $LDFLAGS << ' -Wl,-wrap,memcpy' if linux
-  $LDFLAGS << ' -static-libgcc -static-libstdc++' if RUBY_PLATFORM =~ /linux/
+  $LDFLAGS << ' -static-libgcc -static-libstdc++' if linux
   $LDFLAGS << ' -static' if windows
   
   $CFLAGS << ' -std=c99 '
@@ -104,7 +107,7 @@
   create_makefile(output)
   
   strip_tool = RbConfig::CONFIG['STRIP']
-  strip_tool = 'strip -x' if RUBY_PLATFORM =~ /darwin/
+  strip_tool += ' -x' if darwin
   
   if grpc_config == 'opt'
     File.open('Makefile.new', 'w') do |o|

third_party/rake-compiler-dock/rake_x86_64-darwin/Dockerfile

@@ -0,0 +1 @@
+FROM larskanis/rake-compiler-dock-mri-x86_64-darwin:1.1.0