From 3181f4e530e5465e26463bb56a4dac9baa427016 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Tue, 19 Oct 2021 03:20:24 -0400 Subject: [PATCH] ruby: build native Darwin gems using rake-compiler-dock (#25794) * ruby: use squiggly heredoc for rake-compiler-dock commands * ruby: use `bundler exec` when building native gems * ruby: clean .{bundle,so} from src/ruby/lib when building native gems Failing to remove these files between native builds leads rake-compiler to establish circular dependencies (which may be a bug in rake-compiler, but this feels like an easy and good thing to do, anyway). * ruby: extract linux and darwin RUBY_PLATFORM checks into variables There were already "windows" and "bsd" flags, so let's improve consistency and readability, and set a clear pattern for subsequent flags. * ruby: rely on rake-compiler-dock v1.1.0 to set no_native correctly As of v1.1.0 there's no need to set this explicitly anymore; it will be true whenever the extension is being built in a RCD container. See https://github.com/rake-compiler/rake-compiler-dock/commit/362890d * ruby: add "x86_64-darwin" platform gem Using RCD for this platform unifies the Darwin native gem build process with the Linux native gems, which should help avoid inconsistencies in packaging that result in issues like the missing Ruby 3.0 binaries in #25060. Please note that this change leaves the "universal-darwin" platform native gem untouched, but provides a path forward if the project ever decides to drop "universal" binary support. Related to: - #25429 - #25756 --- Rakefile | 24 +++++++++---------- grpc.gemspec | 2 +- src/ruby/ext/grpc/extconf.rb | 19 ++++++++------- templates/grpc.gemspec.template | 2 +- .../src/ruby/ext/grpc/extconf.rb.template | 19 ++++++++------- .../rake_x86_64-darwin/Dockerfile | 1 + 6 files changed, 36 insertions(+), 31 deletions(-) create mode 100644 third_party/rake-compiler-dock/rake_x86_64-darwin/Dockerfile diff --git a/Rakefile b/Rakefile index faae2106764..5aaafd83c1e 100755 --- a/Rakefile +++ b/Rakefile @@ -23,12 +23,6 @@ end # Add the extension compiler task Rake::ExtensionTask.new('grpc_c', spec) do |ext| - unless RUBY_PLATFORM =~ /darwin/ - # TODO: also set "no_native to true" for mac if possible. As is, - # "no_native" can only be set if the RUBY_PLATFORM doing - # cross-compilation is contained in the "ext.cross_platform" array. - ext.no_native = true - end ext.source_pattern = '**/*.{c,h}' ext.ext_dir = File.join('src', 'ruby', 'ext', 'grpc') ext.lib_dir = File.join('src', 'ruby', 'lib', 'grpc') @@ -36,6 +30,7 @@ Rake::ExtensionTask.new('grpc_c', spec) do |ext| ext.cross_platform = [ 'x86-mingw32', 'x64-mingw32', 'x86_64-linux', 'x86-linux', + 'x86_64-darwin', 'universal-darwin' ] ext.cross_compiling do |spec| @@ -47,6 +42,8 @@ Rake::ExtensionTask.new('grpc_c', spec) do |ext| end end +CLEAN.add "src/ruby/lib/grpc/[0-9].[0-9]", "src/ruby/lib/grpc/grpc_c.{bundle,so}" + # Define the test suites SPEC_SUITES = [ { id: :wrapper, title: 'wrapper layer', files: %w(src/ruby/spec/*.rb) }, @@ -106,14 +103,13 @@ task 'dlls' do env_comp += "CXX=#{opt[:cross]}-g++ " env_comp += "LD=#{opt[:cross]}-gcc " env_comp += "LDXX=#{opt[:cross]}-g++ " - run_rake_compiler opt[:platform], <<-EOT + run_rake_compiler(opt[:platform], <<~EOT) gem update --system --no-document && \ #{env} #{env_comp} make -j`nproc` #{out} && \ #{opt[:cross]}-strip -x -S #{out} && \ cp #{out} #{opt[:out]} EOT end - end desc 'Build the native gem file under rake_compiler_dock' @@ -135,10 +131,11 @@ task 'gem:native' do else Rake::Task['dlls'].execute ['x86-mingw32', 'x64-mingw32'].each do |plat| - run_rake_compiler plat, <<-EOT + run_rake_compiler(plat, <<~EOT) gem update --system --no-document && \ bundle && \ - rake native:#{plat} pkg/#{spec.full_name}-#{plat}.gem pkg/#{spec.full_name}.gem \ + bundle exec rake clean && \ + bundle exec rake native:#{plat} pkg/#{spec.full_name}-#{plat}.gem pkg/#{spec.full_name}.gem \ RUBY_CC_VERSION=#{ruby_cc_versions} \ V=#{verbose} \ GRPC_CONFIG=#{grpc_config} @@ -147,11 +144,12 @@ task 'gem:native' do # Truncate grpc_c.*.ruby files because they're for Windows only. File.truncate('grpc_c.32.ruby', 0) File.truncate('grpc_c.64.ruby', 0) - ['x86_64-linux', 'x86-linux'].each do |plat| - run_rake_compiler plat, <<-EOT + ['x86_64-linux', 'x86-linux', 'x86_64-darwin'].each do |plat| + run_rake_compiler(plat, <<~EOT) gem update --system --no-document && \ bundle && \ - rake native:#{plat} pkg/#{spec.full_name}-#{plat}.gem pkg/#{spec.full_name}.gem \ + bundle exec rake clean && \ + bundle exec rake native:#{plat} pkg/#{spec.full_name}-#{plat}.gem pkg/#{spec.full_name}.gem \ RUBY_CC_VERSION=#{ruby_cc_versions} \ V=#{verbose} \ GRPC_CONFIG=#{grpc_config} diff --git a/grpc.gemspec b/grpc.gemspec index be68619d956..55fa6d8d109 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -41,7 +41,7 @@ Gem::Specification.new do |s| s.add_development_dependency 'simplecov', '~> 0.14.1' s.add_development_dependency 'rake', '~> 13.0' s.add_development_dependency 'rake-compiler', '~> 1.1' - s.add_development_dependency 'rake-compiler-dock', '~> 1.0' + s.add_development_dependency 'rake-compiler-dock', '~> 1.1' s.add_development_dependency 'rspec', '~> 3.6' s.add_development_dependency 'rubocop', '~> 0.49.1' s.add_development_dependency 'signet', '~> 0.7' diff --git a/src/ruby/ext/grpc/extconf.rb b/src/ruby/ext/grpc/extconf.rb index e4b81b4d472..3b6b029b7ed 100644 --- a/src/ruby/ext/grpc/extconf.rb +++ b/src/ruby/ext/grpc/extconf.rb @@ -17,6 +17,9 @@ require 'mkmf' windows = RUBY_PLATFORM =~ /mingw|mswin/ bsd = RUBY_PLATFORM =~ /bsd/ +darwin = RUBY_PLATFORM =~ /darwin/ +linux = RUBY_PLATFORM =~ /linux/ +cross_compiling = ENV['RCD_HOST_RUBY_VERSION'] # set by rake-compiler-dock in build containers grpc_root = File.expand_path(File.join(File.dirname(__FILE__), '../../../..')) @@ -37,17 +40,17 @@ if ENV['LD'].nil? || ENV['LD'].size == 0 ENV['LD'] = ENV['CC'] end -if RUBY_PLATFORM =~ /darwin/ +if darwin && !cross_compiling ENV['AR'] = 'libtool' ENV['ARFLAGS'] = '-o' - end +end ENV['EMBED_OPENSSL'] = 'true' ENV['EMBED_ZLIB'] = 'true' ENV['EMBED_CARES'] = 'true' ENV['ARCH_FLAGS'] = RbConfig::CONFIG['ARCH_FLAG'] -if RUBY_PLATFORM =~ /darwin/ +if darwin && !cross_compiling if RUBY_PLATFORM =~ /arm64/ ENV['ARCH_FLAGS'] = '-arch arm64' else @@ -75,8 +78,8 @@ end $CFLAGS << ' -I' + File.join(grpc_root, 'include') ext_export_file = File.join(grpc_root, 'src', 'ruby', 'ext', 'grpc', 'ext-export') -$LDFLAGS << ' -Wl,--version-script="' + ext_export_file + '.gcc"' if RUBY_PLATFORM =~ /linux/ -$LDFLAGS << ' -Wl,-exported_symbols_list,"' + ext_export_file + '.clang"' if RUBY_PLATFORM =~ /darwin/ +$LDFLAGS << ' -Wl,--version-script="' + ext_export_file + '.gcc"' if linux +$LDFLAGS << ' -Wl,-exported_symbols_list,"' + ext_export_file + '.clang"' if darwin $LDFLAGS << ' ' + File.join(grpc_lib_dir, 'libgrpc.a') unless windows if grpc_config == 'gcov' @@ -88,8 +91,8 @@ if grpc_config == 'dbg' $CFLAGS << ' -O0 -ggdb3' end -$LDFLAGS << ' -Wl,-wrap,memcpy' if RUBY_PLATFORM =~ /linux/ -$LDFLAGS << ' -static-libgcc -static-libstdc++' if RUBY_PLATFORM =~ /linux/ +$LDFLAGS << ' -Wl,-wrap,memcpy' if linux +$LDFLAGS << ' -static-libgcc -static-libstdc++' if linux $LDFLAGS << ' -static' if windows $CFLAGS << ' -std=c99 ' @@ -102,7 +105,7 @@ puts 'Generating Makefile for ' + output create_makefile(output) strip_tool = RbConfig::CONFIG['STRIP'] -strip_tool = 'strip -x' if RUBY_PLATFORM =~ /darwin/ +strip_tool += ' -x' if darwin if grpc_config == 'opt' File.open('Makefile.new', 'w') do |o| diff --git a/templates/grpc.gemspec.template b/templates/grpc.gemspec.template index ac543af7a64..d7f625f70cf 100644 --- a/templates/grpc.gemspec.template +++ b/templates/grpc.gemspec.template @@ -43,7 +43,7 @@ s.add_development_dependency 'simplecov', '~> 0.14.1' s.add_development_dependency 'rake', '~> 13.0' s.add_development_dependency 'rake-compiler', '~> 1.1' - s.add_development_dependency 'rake-compiler-dock', '~> 1.0' + s.add_development_dependency 'rake-compiler-dock', '~> 1.1' s.add_development_dependency 'rspec', '~> 3.6' s.add_development_dependency 'rubocop', '~> 0.49.1' s.add_development_dependency 'signet', '~> 0.7' diff --git a/templates/src/ruby/ext/grpc/extconf.rb.template b/templates/src/ruby/ext/grpc/extconf.rb.template index 1130ba8bbbc..03a50d736c8 100644 --- a/templates/src/ruby/ext/grpc/extconf.rb.template +++ b/templates/src/ruby/ext/grpc/extconf.rb.template @@ -19,6 +19,9 @@ windows = RUBY_PLATFORM =~ /mingw|mswin/ bsd = RUBY_PLATFORM =~ /bsd/ + darwin = RUBY_PLATFORM =~ /darwin/ + linux = RUBY_PLATFORM =~ /linux/ + cross_compiling = ENV['RCD_HOST_RUBY_VERSION'] # set by rake-compiler-dock in build containers grpc_root = File.expand_path(File.join(File.dirname(__FILE__), '../../../..')) @@ -39,17 +42,17 @@ ENV['LD'] = ENV['CC'] end - if RUBY_PLATFORM =~ /darwin/ + if darwin && !cross_compiling ENV['AR'] = 'libtool' ENV['ARFLAGS'] = '-o' - end + end ENV['EMBED_OPENSSL'] = 'true' ENV['EMBED_ZLIB'] = 'true' ENV['EMBED_CARES'] = 'true' ENV['ARCH_FLAGS'] = RbConfig::CONFIG['ARCH_FLAG'] - if RUBY_PLATFORM =~ /darwin/ + if darwin && !cross_compiling if RUBY_PLATFORM =~ /arm64/ ENV['ARCH_FLAGS'] = '-arch arm64' else @@ -77,8 +80,8 @@ $CFLAGS << ' -I' + File.join(grpc_root, 'include') ext_export_file = File.join(grpc_root, 'src', 'ruby', 'ext', 'grpc', 'ext-export') - $LDFLAGS << ' -Wl,--version-script="' + ext_export_file + '.gcc"' if RUBY_PLATFORM =~ /linux/ - $LDFLAGS << ' -Wl,-exported_symbols_list,"' + ext_export_file + '.clang"' if RUBY_PLATFORM =~ /darwin/ + $LDFLAGS << ' -Wl,--version-script="' + ext_export_file + '.gcc"' if linux + $LDFLAGS << ' -Wl,-exported_symbols_list,"' + ext_export_file + '.clang"' if darwin $LDFLAGS << ' ' + File.join(grpc_lib_dir, 'libgrpc.a') unless windows if grpc_config == 'gcov' @@ -90,8 +93,8 @@ $CFLAGS << ' -O0 -ggdb3' end - $LDFLAGS << ' -Wl,-wrap,memcpy' if RUBY_PLATFORM =~ /linux/ - $LDFLAGS << ' -static-libgcc -static-libstdc++' if RUBY_PLATFORM =~ /linux/ + $LDFLAGS << ' -Wl,-wrap,memcpy' if linux + $LDFLAGS << ' -static-libgcc -static-libstdc++' if linux $LDFLAGS << ' -static' if windows $CFLAGS << ' -std=c99 ' @@ -104,7 +107,7 @@ create_makefile(output) strip_tool = RbConfig::CONFIG['STRIP'] - strip_tool = 'strip -x' if RUBY_PLATFORM =~ /darwin/ + strip_tool += ' -x' if darwin if grpc_config == 'opt' File.open('Makefile.new', 'w') do |o| diff --git a/third_party/rake-compiler-dock/rake_x86_64-darwin/Dockerfile b/third_party/rake-compiler-dock/rake_x86_64-darwin/Dockerfile new file mode 100644 index 00000000000..5c7bb861000 --- /dev/null +++ b/third_party/rake-compiler-dock/rake_x86_64-darwin/Dockerfile @@ -0,0 +1 @@ +FROM larskanis/rake-compiler-dock-mri-x86_64-darwin:1.1.0