Merge pull request #23344 from veblush/ruby-docker-ml

Ruby docker image based on manylinux2010
pull/23530/head
Esun Kim 4 years ago committed by GitHub
commit 47c6c826f7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 22
      Rakefile
  2. 46
      third_party/rake-compiler-dock/README.md
  3. 30
      third_party/rake-compiler-dock/build/patchelf_gem.sh
  4. 24
      third_party/rake-compiler-dock/build/patches/ruby-2.5.7/no_sendfile.patch
  5. 24
      third_party/rake-compiler-dock/build/patches/ruby-2.7.0/no_sendfile.patch
  6. 32
      third_party/rake-compiler-dock/build/patches2/hoe-3.20.0/0001-Load-encrypted-private-key-using-ENV-GEM_PRIVATE_KEY.patch
  7. 44
      third_party/rake-compiler-dock/build/patches2/rake-compiler-1.1.0/0001-Fix-determining-of-ruby-versions-in-rake-native-gem.patch
  8. 64
      third_party/rake-compiler-dock/build/patches2/rake-compiler-1.1.0/0001-cross-ruby-remove-needless-Makefile.in-preparation.patch
  9. 26
      third_party/rake-compiler-dock/build/patches2/rake-compiler-1.1.0/0002-Extend-mingw-search-pattern-to-find-x86_64-gcc.patch
  10. 29
      third_party/rake-compiler-dock/build/patches2/rake-compiler-1.1.0/0002-Strip-cross-built-shared-library-files-while-linking.patch
  11. 214
      third_party/rake-compiler-dock/build/patches2/rake-compiler-1.1.0/0003-Allow-building-of-cross-rubies-in-parallel.patch
  12. 28
      third_party/rake-compiler-dock/build/patches2/rake-compiler-1.1.0/0004-Enable-build-of-static-libruby.patch
  13. 15
      third_party/rake-compiler-dock/build/patches2/ruby-2.7.0/ruby2_keywords.patch
  14. 45
      third_party/rake-compiler-dock/build/sigfw.c
  15. 47
      third_party/rake-compiler-dock/push_testing_images.sh
  16. 92
      third_party/rake-compiler-dock/rake_x86-linux/Dockerfile
  17. 92
      third_party/rake-compiler-dock/rake_x86_64-linux/Dockerfile
  18. 6
      tools/distrib/rake_compiler_docker_image.rb
  19. 2
      tools/dockerfile/push_testing_images.sh

@ -121,6 +121,7 @@ task 'gem:native' do
verbose = ENV['V'] || '0'
grpc_config = ENV['GRPC_CONFIG'] || 'opt'
ruby_cc_versions = '2.7.0:2.6.0:2.5.0:2.4.0:2.3.0'
if RUBY_PLATFORM =~ /darwin/
FileUtils.touch 'grpc_c.32.ruby'
@ -130,17 +131,28 @@ task 'gem:native' do
"invoked on macos with ruby #{RUBY_VERSION}. The ruby macos artifact " \
"build should be running on ruby 2.5."
end
system "rake cross native gem RUBY_CC_VERSION=2.7.0:2.6.0:2.5.0:2.4.0:2.3.0 V=#{verbose} GRPC_CONFIG=#{grpc_config}"
system "rake cross native gem RUBY_CC_VERSION=#{ruby_cc_versions} V=#{verbose} GRPC_CONFIG=#{grpc_config}"
else
Rake::Task['dlls'].execute
['x86-mingw32', 'x64-mingw32', 'x86_64-linux', 'x86-linux'].each do |plat|
['x86-mingw32', 'x64-mingw32'].each do |plat|
run_rake_compiler plat, <<-EOT
gem update --system --no-document && \
bundle && \
rake native:#{plat} pkg/#{spec.full_name}-#{plat}.gem pkg/#{spec.full_name}.gem \
RUBY_CC_VERSION=2.7.0:2.6.0:2.5.0:2.4.0:2.3.0 \
V=#{verbose} \
GRPC_CONFIG=#{grpc_config}
RUBY_CC_VERSION=#{ruby_cc_versions} V=#{verbose} GRPC_CONFIG=#{grpc_config}
EOT
end
# Truncate grpc_c.*.ruby files because they're for Windows only.
File.truncate('grpc_c.32.ruby', 0)
File.truncate('grpc_c.64.ruby', 0)
['x86_64-linux', 'x86-linux'].each do |plat|
run_rake_compiler plat, <<-EOT
gem update --system --no-document && \
bundle && \
rake native:#{plat} pkg/#{spec.full_name}-#{plat}.gem pkg/#{spec.full_name}.gem \
RUBY_CC_VERSION=#{ruby_cc_versions} V=#{verbose} GRPC_CONFIG=#{grpc_config} &&
sudo chmod -R a+rw pkg &&
patchelf_gem.sh pkg/#{spec.full_name}-#{plat}.gem
EOT
end
end

@ -0,0 +1,46 @@
rake-compiler-dock for gRPC
===================================
This has customized docker images for Ruby based on
[rake-compiler-dock](https://github.com/rake-compiler/rake-compiler-dock).
gRPC needs four docker images to build Ruby artifacts;
- rake_x64-linux: Linux / 32bit
- rake_x86_64-linux: Linux / 64bit
- rake_x86-mingw32: Windows / 32bit
- rake_x64-mingw32: Windows / 64bit
### Customization
#### Linux
The linux docker images of `rake-compiler-dock` are based on Ubuntu, which is enough for
most cases but it becomes hard to keep it compataible for some conservative Linux distrubitions
such as CentOS 6 because Ubuntu uses more modern libraries than them.
As a result, generated artifacts sometimes cannot run on CentOS 6 due to missing dependencies.
This can be easily addressed by using CentOS 6 based docker images such as
[dockcross manylinux2010](https://github.com/dockcross/dockcross), which was invented
to handle the very same problem of Python. By using the same solution,
Ruby can have the simple way of building more portable artifacts.
This idea is summarized in
[rake-compiler-dock#33](https://github.com/rake-compiler/rake-compiler-dock/issues/33).
These two new docker images; `rake_x64-linux` and `rake_x86_64-linux` are based on
[Dockerfile.mri.erb](https://github.com/rake-compiler/rake-compiler-dock/blob/master/Dockerfile.mri.erb)
with following customizations;
- Changing the base image from `ubuntu:16.04` to `dockcross/manylinux2010`
- Removing rvm account due to the complexity of having the same thing on `manylinux2010`
(mainly due to the limit of `gosu` handling groups)
- Removing cross compiling setup for x86 because `manylinux2010-x86` already did it.
(like cross compilers for x86 and mk_i686)
- Removing glibc hack because `manylinux2010` doesn't needit.
- Adding `patchelf_gem.sh` to trim the unnecessary dependency of `libcrypt.so.2`.
Without this, artifacts for Ruby 2.3 to 2.5 happens to have a `libcrypt.so.2` link although
it doesn't have any external symbols from it.
#### Windows
Windows docker images are almost identical to `rake-compiler-dock` but with some exception;
- Renaming `gettimeofday` to `rb_gettimeofday` in `win32.h`

@ -0,0 +1,30 @@
#/bin/bash
set -ex
if [ -z $1 ] ; then
echo "Gem file needed!" && exit 1;
fi
GEM=$1
GEM_FILENAME=$(basename -- "$GEM")
GEM_NAME="${GEM_FILENAME%.gem}"
# Extract all files onto a temporary directory
TMPDIR=$(mktemp -d -t gem-XXXXXXXXXX)
gem unpack $GEM --target=$TMPDIR
gem spec $GEM --ruby > ${TMPDIR}/${GEM_NAME}/${GEM_NAME}.gemspec
# Run patchelf to all so files to strip out unnecessary libcrypt.so.2 dependency
find $TMPDIR/${GEM_NAME} -name "*.so" \
-printf '%p\n' \
-exec patchelf --remove-needed libcrypt.so.2 {} \;
# Rebuild the gem again with modified so files
pushd $TMPDIR/${GEM_NAME}
gem build ${GEM_NAME}.gemspec
popd
# Keep the new result
mv $TMPDIR/${GEM_NAME}/${GEM_NAME}.gem $GEM
rm -rf $TMPDIR

@ -0,0 +1,24 @@
diff --git a/configure b/configure
index ebe3d8c..a336b73 100755
--- a/configure
+++ b/configure
@@ -18943,7 +18943,6 @@ do :
ac_fn_c_check_func "$LINENO" "sendfile" "ac_cv_func_sendfile"
if test "x$ac_cv_func_sendfile" = xyes; then :
cat >>confdefs.h <<_ACEOF
-#define HAVE_SENDFILE 1
_ACEOF
fi
diff --git a/io.c b/io.c
index 82c5940538..a8d3661ec8 100644
--- a/io.c
+++ b/io.c
@@ -10721,7 +10721,6 @@ nogvl_copy_stream_wait_write(struct copy_stream_struct *stp)
}
#if defined HAVE_COPY_FILE_RANGE || (defined __linux__ && defined __NR_copy_file_range)
-# define USE_COPY_FILE_RANGE
#endif
#ifdef USE_COPY_FILE_RANGE

@ -0,0 +1,24 @@
diff --git a/configure b/configure
index ebe3d8c..a336b73 100755
--- a/configure
+++ b/configure
@@ -18943,7 +18943,6 @@ do :
ac_fn_c_check_func "$LINENO" "sendfile" "ac_cv_func_sendfile"
if test "x$ac_cv_func_sendfile" = xyes; then :
cat >>confdefs.h <<_ACEOF
-#define HAVE_SENDFILE 1
_ACEOF
fi
diff --git a/io.c b/io.c
index 82c5940538..a8d3661ec8 100644
--- a/io.c
+++ b/io.c
@@ -10721,7 +10721,6 @@ nogvl_copy_stream_wait_write(struct copy_stream_struct *stp)
}
#if defined HAVE_COPY_FILE_RANGE || (defined __linux__ && defined __NR_copy_file_range)
-# define USE_COPY_FILE_RANGE
#endif
#ifdef USE_COPY_FILE_RANGE

@ -0,0 +1,32 @@
From ecc5669223457ceaba6bd94c2947ae99ddfa8f10 Mon Sep 17 00:00:00 2001
From: Lars Kanis <lars@greiz-reinsdorf.de>
Date: Fri, 27 Dec 2019 15:57:07 +0100
Subject: [PATCH] Load encrypted private key using
ENV['GEM_PRIVATE_KEY_PASSPHRASE'] as passphrase.
This way the password can be avoided to be entered several times when building multiple gems.
It also allows parallel builds which are incompatible to interactive password prompts.
The same feature has been implemented in rubygems:
https://github.com/rubygems/rubygems/commit/c1a114396fcec124d3feabf74708b4bdec02eac3
---
lib/hoe/signing.rb | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/lib/hoe/signing.rb b/lib/hoe/signing.rb
index 4833c55..11c8c2f 100644
--- a/lib/hoe/signing.rb
+++ b/lib/hoe/signing.rb
@@ -70,7 +70,8 @@ module Hoe::Signing
end
if signing_key and cert_chain then
- spec.signing_key = OpenSSL::PKey::RSA.new File.read signing_key
+ passphrase = ENV['GEM_PRIVATE_KEY_PASSPHRASE']
+ spec.signing_key = OpenSSL::PKey::RSA.new(File.read(signing_key), passphrase)
spec.cert_chain = cert_chain
end
end
--
2.20.1

@ -0,0 +1,44 @@
From 365843bfe0bfc3f1c1b11e81e24ac7b6c7942bdf Mon Sep 17 00:00:00 2001
From: Lars Kanis <lars@greiz-reinsdorf.de>
Date: Fri, 27 Dec 2019 18:18:59 +0100
Subject: [PATCH] Fix determining of ruby versions in "rake native gem"
"rake native gem" without "cross" didn't set the ruby version constraint.
Instead it failed with NoMethodError like so:
/ffi $ rake native gem
no configuration section for specified version of Ruby (rbconfig-i386-mingw32-2.6.3)
no configuration section for specified version of Ruby (rbconfig-x64-mingw32-2.6.3)
install -c build/x86_64-linux/ffi_c/2.6.3/ffi_c.so lib/ffi_c.so
cp build/x86_64-linux/ffi_c/2.6.3/ffi_c.so build/x86_64-linux/stage/lib/ffi_c.so
rake aborted!
NoMethodError: undefined method `split' for nil:NilClass
/home/lars/.rvm/gems/ruby-2.6.3/gems/rake-compiler-1.0.9/lib/rake/extensiontask.rb:515:in `ruby_api_version'
/home/lars/.rvm/gems/ruby-2.6.3/gems/rake-compiler-1.0.9/lib/rake/extensiontask.rb:262:in `block in define_native_tasks'
/home/lars/.rvm/gems/ruby-2.6.3/gems/rake-12.3.3/exe/rake:27:in `<top (required)>'
/home/lars/.rvm/gems/ruby-2.6.3/bin/ruby_executable_hooks:24:in `eval'
/home/lars/.rvm/gems/ruby-2.6.3/bin/ruby_executable_hooks:24:in `<main>'
Tasks: TOP => native => native:x86_64-linux => native:ffi:x86_64-linux
(See full trace by running task with --trace)
---
lib/rake/extensiontask.rb | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/lib/rake/extensiontask.rb b/lib/rake/extensiontask.rb
index d85b1a3..cb1ea12 100644
--- a/lib/rake/extensiontask.rb
+++ b/lib/rake/extensiontask.rb
@@ -242,6 +242,10 @@ Java extension should be preferred.
# lib_path
lib_path = lib_dir
+ # Update compiled platform/version combinations
+ ruby_versions = (@ruby_versions_per_platform[platf] ||= [])
+ ruby_versions << ruby_ver
+
# create 'native:gem_name' and chain it to 'native' task
unless Rake::Task.task_defined?("native:#{@gem_spec.name}:#{platf}")
task "native:#{@gem_spec.name}:#{platf}" do |t|
--
2.20.1

@ -0,0 +1,64 @@
From d76da276ba556bdefdddcc775b7f7907b8e1273e Mon Sep 17 00:00:00 2001
From: Sutou Kouhei <kou@clear-code.com>
Date: Wed, 25 Dec 2019 06:28:56 +0900
Subject: [PATCH 1/5] cross-ruby: remove needless Makefile.in preparation
ALT_SEPARATOR doesn't exist in Makefile.in since Ruby 1.9.2:
https://github.com/ruby/ruby/commit/7c7690045870396816624bf57775eb29e6a478fd
---
tasks/bin/cross-ruby.rake | 32 +-------------------------------
1 file changed, 1 insertion(+), 31 deletions(-)
diff --git a/tasks/bin/cross-ruby.rake b/tasks/bin/cross-ruby.rake
index 37dd220..c9b0bbb 100644
--- a/tasks/bin/cross-ruby.rake
+++ b/tasks/bin/cross-ruby.rake
@@ -96,36 +96,6 @@ file source_dir => ["#{USER_HOME}/sources/#{source_file}"] do |t|
end
end
-# backup makefile.in
-if RUBY_CC_VERSION >= "ruby-2.7.0"
- makefile_in = "#{source_dir}/template/Makefile.in"
-else
- makefile_in = "#{source_dir}/Makefile.in"
-end
-makefile_in_bak = "#{makefile_in}.bak"
-file makefile_in_bak => [source_dir] do |t|
- cp makefile_in, makefile_in_bak
-end
-
-# correct the makefiles
-file makefile_in => [makefile_in_bak] do |t|
- content = File.open(makefile_in_bak, 'rb') { |f| f.read }
-
- out = ""
-
- content.each_line do |line|
- if line =~ /^\s*ALT_SEPARATOR =/
- out << "\t\t ALT_SEPARATOR = \"\\\\\\\\\"; \\\n"
- else
- out << line
- end
- end
-
- when_writing("Patching Makefile.in") {
- File.open(makefile_in, 'wb') { |f| f.write(out) }
- }
-end
-
task :mingw32 do
unless MINGW_HOST then
warn "You need to install mingw32 cross compile functionality to be able to continue."
@@ -135,7 +105,7 @@ task :mingw32 do
end
# generate the makefile in a clean build location
-file "#{build_dir}/Makefile" => [build_dir, makefile_in] do |t|
+file "#{build_dir}/Makefile" => [build_dir, source_dir] do |t|
options = [
"--host=#{MINGW_HOST}",
--
2.20.1

@ -0,0 +1,26 @@
From 07b1a4909fa59324d309bb8a46867cacba97cafd Mon Sep 17 00:00:00 2001
From: Lars Kanis <lars@greiz-reinsdorf.de>
Date: Thu, 26 Dec 2019 18:31:25 +0100
Subject: [PATCH] Extend mingw search pattern to find x86_64 gcc
The previous pattern only recognized 32 bit compiler versions.
---
lib/rake/extensioncompiler.rb | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/rake/extensioncompiler.rb b/lib/rake/extensioncompiler.rb
index b3d336b..75f4694 100644
--- a/lib/rake/extensioncompiler.rb
+++ b/lib/rake/extensioncompiler.rb
@@ -37,7 +37,7 @@ module Rake
paths = ENV['PATH'].split(File::PATH_SEPARATOR)
# the pattern to look into (captures *nix and windows executables)
- pattern = "{mingw32-,i?86*mingw*}gcc{,.*}"
+ pattern = "{mingw32-,i?86*mingw*,x86*mingw*}gcc{,.*}"
@mingw_gcc_executable = paths.find do |path|
# cleanup paths before globbing
--
2.20.1

@ -0,0 +1,29 @@
From 13e33405d4e51a52402e1e37e7e2df7f1f32892d Mon Sep 17 00:00:00 2001
From: Lars Kanis <lars@greiz-reinsdorf.de>
Date: Thu, 26 Dec 2019 19:18:02 +0100
Subject: [PATCH 2/2] Strip cross built shared library files while linking
The .so files of extensions are often stripped after compilation per task in a Rakefile.
However this can be omitted, if the cross ruby version is built with stripping enabled.
Stripping the files a second time doesn't make a difference then.
---
tasks/bin/cross-ruby.rake | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/tasks/bin/cross-ruby.rake b/tasks/bin/cross-ruby.rake
index 2a73be7..a9eb16b 100644
--- a/tasks/bin/cross-ruby.rake
+++ b/tasks/bin/cross-ruby.rake
@@ -115,7 +115,8 @@ file "#{build_dir}/Makefile" => [build_dir, source_dir] do |t|
"--build=#{RUBY_BUILD}",
'--enable-shared',
'--disable-install-doc',
- '--with-ext='
+ '--with-ext=',
+ 'LDFLAGS=-pipe -s',
]
# Force Winsock2 for Ruby 1.8, 1.9 defaults to it
--
2.20.1

@ -0,0 +1,214 @@
From 01fd7364bdcb37ac8709ffa48388b5cbe6478211 Mon Sep 17 00:00:00 2001
From: Lars Kanis <lars@greiz-reinsdorf.de>
Date: Fri, 10 Jan 2020 23:57:18 +0100
Subject: [PATCH] Allow building of cross rubies in parallel
Rubies can be build like so:
rake-compiler cross-ruby VERSION=2.7.0:2.6.0 HOST=x86_64-w64-mingw32:i686-w64-mingw32
This builds the cross product of all ":" separated ruby and host versions.
To force sequential builds add option "-j1".
---
tasks/bin/cross-ruby.rake | 161 +++++++++++++++++++-------------------
1 file changed, 80 insertions(+), 81 deletions(-)
diff --git a/tasks/bin/cross-ruby.rake b/tasks/bin/cross-ruby.rake
index 278541c..8b88025 100644
--- a/tasks/bin/cross-ruby.rake
+++ b/tasks/bin/cross-ruby.rake
@@ -41,106 +41,105 @@ end
require 'rake/extensioncompiler'
MAKE = ENV['MAKE'] || %w[gmake make].find { |c| system("#{c} -v > /dev/null 2>&1") }
-USER_HOME = File.expand_path("~/.rake-compiler")
-RUBY_CC_VERSION = "ruby-" << ENV.fetch("VERSION", "1.8.7-p371")
+USER_HOME = File.realpath(File.expand_path("~/.rake-compiler"))
RUBY_SOURCE = ENV['SOURCE']
RUBY_BUILD = RbConfig::CONFIG["host"]
-# grab the major "1.8" or "1.9" part of the version number
-MAJOR = RUBY_CC_VERSION.match(/.*-(\d.\d).\d/)[1]
-
-# Use Rake::ExtensionCompiler helpers to find the proper host
-MINGW_HOST = ENV['HOST'] || Rake::ExtensionCompiler.mingw_host
-MINGW_TARGET = MINGW_HOST.gsub('msvc', '')
-
# Unset any possible variable that might affect compilation
["CC", "CXX", "CPPFLAGS", "LDFLAGS", "RUBYOPT"].each do |var|
ENV.delete(var)
end
-source_dir = "#{USER_HOME}/sources/#{RUBY_CC_VERSION}"
-build_dir = "#{USER_HOME}/builds/#{MINGW_HOST}/#{RUBY_CC_VERSION}"
-
-# define a location where sources will be stored
-directory source_dir
-directory build_dir
-
-# clean intermediate files and folders
-CLEAN.include(source_dir)
-CLEAN.include(build_dir)
-
-# remove the final products and sources
-CLOBBER.include("#{USER_HOME}/sources")
-CLOBBER.include("#{USER_HOME}/builds")
-CLOBBER.include("#{USER_HOME}/ruby/#{MINGW_HOST}/#{RUBY_CC_VERSION}")
-CLOBBER.include("#{USER_HOME}/config.yml")
+RUBY_CC_VERSIONS = ENV.fetch("VERSION", "1.8.7-p371")
+RUBY_CC_VERSIONS.split(":").each do |ruby_cc_version|
+ ruby_cc_version = "ruby-" + ruby_cc_version
+ # grab the major "1.8" or "1.9" part of the version number
+ major = ruby_cc_version.match(/.*-(\d.\d).\d/)[1]
+
+ # define a location where sources will be stored
+ source_dir = "#{USER_HOME}/sources/#{ruby_cc_version}"
+ directory source_dir
+ # clean intermediate files and folders
+ CLEAN.include(source_dir)
+
+ # remove the final products and sources
+ CLOBBER.include("#{USER_HOME}/sources")
+ CLOBBER.include("#{USER_HOME}/builds")
+ CLOBBER.include("#{USER_HOME}/config.yml")
+
+ # Extract the sources
+ source_file = RUBY_SOURCE ? RUBY_SOURCE.split('/').last : "#{ruby_cc_version}.tar.bz2"
+ file source_dir => ["#{USER_HOME}/sources/#{source_file}"] do |t|
+ t.prerequisites.each { |f| sh "tar xf #{File.basename(f)}", chdir: File.dirname(t.name) }
+ end
-# ruby source file should be stored there
-file "#{USER_HOME}/sources/#{RUBY_CC_VERSION}.tar.bz2" => ["#{USER_HOME}/sources"] do |t|
- # download the source file using wget or curl
- chdir File.dirname(t.name) do
+ # ruby source file should be stored there
+ file "#{USER_HOME}/sources/#{ruby_cc_version}.tar.bz2" => ["#{USER_HOME}/sources"] do |t|
+ # download the source file using wget or curl
if RUBY_SOURCE
url = RUBY_SOURCE
else
- url = "http://cache.ruby-lang.org/pub/ruby/#{MAJOR}/#{File.basename(t.name)}"
+ url = "http://cache.ruby-lang.org/pub/ruby/#{major}/#{File.basename(t.name)}"
end
- sh "wget #{url} || curl -O #{url}"
+ sh "wget #{url} || curl -O #{url}", chdir: File.dirname(t.name)
end
-end
-
-# Extract the sources
-source_file = RUBY_SOURCE ? RUBY_SOURCE.split('/').last : "#{RUBY_CC_VERSION}.tar.bz2"
-file source_dir => ["#{USER_HOME}/sources/#{source_file}"] do |t|
- chdir File.dirname(t.name) do
- t.prerequisites.each { |f| sh "tar xf #{File.basename(f)}" }
- end
-end
-task :mingw32 do
- unless MINGW_HOST then
- warn "You need to install mingw32 cross compile functionality to be able to continue."
- warn "Please refer to your distribution/package manager documentation about installation."
- fail
- end
-end
+ # Create tasks for each host out of the ":" separated hosts list in the HOST variable.
+ # These tasks are processed in parallel as dependencies to the "install" task.
+ mingw_hosts = ENV['HOST'] || Rake::ExtensionCompiler.mingw_host
+ mingw_hosts.split(":").each do |mingw_host|
+
+ # Use Rake::ExtensionCompiler helpers to find the proper host
+ mingw_target = mingw_host.gsub('msvc', '')
+
+ # define a location where built files for each host will be stored
+ build_dir = "#{USER_HOME}/builds/#{mingw_host}/#{ruby_cc_version}"
+ directory build_dir
+ install_dir = "#{USER_HOME}/ruby/#{mingw_host}/#{ruby_cc_version}"
+
+ # clean intermediate files and folders
+ CLEAN.include(build_dir)
+ CLOBBER.include(install_dir)
+
+ task :mingw32 do
+ unless mingw_host then
+ warn "You need to install mingw32 cross compile functionality to be able to continue."
+ warn "Please refer to your distribution/package manager documentation about installation."
+ fail
+ end
+ end
-# generate the makefile in a clean build location
-file "#{build_dir}/Makefile" => [build_dir, source_dir] do |t|
-
- options = [
- "--host=#{MINGW_HOST}",
- "--target=#{MINGW_TARGET}",
- "--build=#{RUBY_BUILD}",
- '--enable-shared',
- '--disable-install-doc',
- '--with-ext=',
- 'LDFLAGS=-pipe -s',
- ]
-
- # Force Winsock2 for Ruby 1.8, 1.9 defaults to it
- options << "--with-winsock2" if MAJOR == "1.8"
-
- chdir File.dirname(t.name) do
- prefix = File.expand_path("../../../ruby/#{MINGW_HOST}/#{RUBY_CC_VERSION}")
- options << "--prefix=#{prefix}"
- sh File.expand_path("../../../sources/#{RUBY_CC_VERSION}/configure"), *options
- end
-end
+ # generate the makefile in a clean build location
+ file "#{build_dir}/Makefile" => [build_dir, source_dir] do |t|
+
+ options = [
+ "--host=#{mingw_host}",
+ "--target=#{mingw_target}",
+ "--build=#{RUBY_BUILD}",
+ '--enable-shared',
+ '--disable-install-doc',
+ '--with-ext=',
+ 'LDFLAGS=-pipe -s',
+ ]
+
+ # Force Winsock2 for Ruby 1.8, 1.9 defaults to it
+ options << "--with-winsock2" if major == "1.8"
+ options << "--prefix=#{install_dir}"
+ sh File.expand_path("#{USER_HOME}/sources/#{ruby_cc_version}/configure"), *options, chdir: File.dirname(t.name)
+ end
-# make
-file "#{build_dir}/ruby.exe" => ["#{build_dir}/Makefile"] do |t|
- chdir File.dirname(t.prerequisites.first) do
- sh MAKE
- end
-end
+ # make
+ file "#{build_dir}/ruby.exe" => ["#{build_dir}/Makefile"] do |t|
+ sh MAKE, chdir: File.dirname(t.prerequisites.first)
+ end
-# make install
-file "#{USER_HOME}/ruby/#{MINGW_HOST}/#{RUBY_CC_VERSION}/bin/ruby.exe" => ["#{build_dir}/ruby.exe"] do |t|
- chdir File.dirname(t.prerequisites.first) do
- sh "#{MAKE} install"
+ # make install
+ file "#{USER_HOME}/ruby/#{mingw_host}/#{ruby_cc_version}/bin/ruby.exe" => ["#{build_dir}/ruby.exe"] do |t|
+ sh "#{MAKE} install", chdir: File.dirname(t.prerequisites.first)
+ end
+ multitask :install => ["#{USER_HOME}/ruby/#{mingw_host}/#{ruby_cc_version}/bin/ruby.exe"]
end
end
-task :install => ["#{USER_HOME}/ruby/#{MINGW_HOST}/#{RUBY_CC_VERSION}/bin/ruby.exe"]
desc "Update rake-compiler list of installed Ruby versions"
task 'update-config' do
@@ -187,5 +186,5 @@ task :default do
Rake.application.display_tasks_and_comments
end
-desc "Build #{RUBY_CC_VERSION} suitable for cross-platform development."
+desc "Build rubies suitable for cross-platform development."
task 'cross-ruby' => [:mingw32, :install, 'update-config']
--
2.20.1

@ -0,0 +1,28 @@
From 0b929b7027eb592c0971a400184f1dfa5a0bce7c Mon Sep 17 00:00:00 2001
From: Lars Kanis <lars@greiz-reinsdorf.de>
Date: Thu, 26 Dec 2019 19:13:48 +0100
Subject: [PATCH 1/2] Enable build of static libruby
This is used in rake-compiler-dock for linux targets.
The static libruby is postprocessed in a special way.
See rake-compiler-dock's Dockerfile.
---
tasks/bin/cross-ruby.rake | 2 ++
1 file changed, 2 insertions(+)
diff --git a/tasks/bin/cross-ruby.rake b/tasks/bin/cross-ruby.rake
index c9b0bbb..2a73be7 100644
--- a/tasks/bin/cross-ruby.rake
+++ b/tasks/bin/cross-ruby.rake
@@ -108,6 +108,8 @@ end
file "#{build_dir}/Makefile" => [build_dir, source_dir] do |t|
options = [
+ '--enable-static',
+ '--enable-install-static-library',
"--host=#{mingw_host}",
"--target=#{mingw_target}",
"--build=#{RUBY_BUILD}",
--
2.20.1

@ -0,0 +1,15 @@
diff --git a/template/fake.rb.in b/template/fake.rb.in
index f1da719ecb..e0b649fb28 100644
--- a/template/fake.rb.in
+++ b/template/fake.rb.in
@@ -27,6 +27,10 @@ case "$0" in /*) r=-r"$0";; *) r=-r"./$0";; esac
exec $ruby "$r" "$@"
=end
=baseruby
+<% if RUBY_VERSION < "2.7" %>
+def ruby2_keywords(*)
+end
+<% end %>
class Object
remove_const :CROSS_COMPILING if defined?(CROSS_COMPILING)
CROSS_COMPILING = RUBY_PLATFORM

@ -0,0 +1,45 @@
/*
* This program handles SIGINT and forwards it to another process.
* It is intended to be run as PID 1.
*
* Docker starts processes with "docker run" as PID 1.
* On Linux, the default signal handler for PID 1 ignores any signals.
* Therefore Ctrl-C aka SIGINT is ignored per default.
*/
#include <unistd.h>
#include <stdio.h>
#include <signal.h>
#include <sys/types.h>
#include <sys/wait.h>
int pid = 0;
void
handle_sigint (int signum)
{
if(pid)
kill(pid, SIGINT);
}
int main(int argc, char *argv[]){
struct sigaction new_action;
int status = -1;
/* Set up the structure to specify the new action. */
new_action.sa_handler = handle_sigint;
sigemptyset (&new_action.sa_mask);
new_action.sa_flags = 0;
sigaction (SIGINT, &new_action, (void*)0);
pid = fork();
if(pid){
wait(&status);
return WEXITSTATUS(status);
}else{
status = execvp(argv[1], &argv[1]);
perror("exec");
return status;
}
}

@ -0,0 +1,47 @@
#!/bin/bash
# Copyright 2020 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Builds selected testing docker images and pushes them to dockerhub.
# Useful for testing environments where it's impractical (or impossible)
# to rely on docker images being cached locally after they've been built
# for the first time (which might be costly especially for some images).
# NOTE: gRPC docker images intended to be used by end users are NOT
# pushed using this script (they're built automatically by dockerhub).
# This script is only for "internal" images we use when testing gRPC.
set -ex
cd $(dirname $0)
DOCKERHUB_ORGANIZATION=grpctesting
for NAME in rake_x86_64-linux rake_x86-linux rake_x64-mingw32 rake_x86-mingw32
do
# Generate image name based on Dockerfile checksum. That works well as long
# as can count on dockerfiles being written in a way that changing the logical
# contents of the docker image always changes the SHA (e.g. using "ADD file"
# cmd in the dockerfile in not ok as contents of the added file will not be
# reflected in the SHA).
DOCKER_IMAGE_NAME=${NAME}_$(sha1sum ${NAME}/Dockerfile | cut -f1 -d\ )
# skip the image if it already exists in the repo
curl --silent -f -lSL https://registry.hub.docker.com/v2/repositories/${DOCKERHUB_ORGANIZATION}/${DOCKER_IMAGE_NAME}/tags/latest > /dev/null \
&& continue
docker build -t ${DOCKERHUB_ORGANIZATION}/${DOCKER_IMAGE_NAME} -f ${NAME}/Dockerfile .
# "docker login" needs to be run in advance
docker push ${DOCKERHUB_ORGANIZATION}/${DOCKER_IMAGE_NAME}
done

@ -1 +1,91 @@
FROM larskanis/rake-compiler-dock-mri-x86-linux:1.0.0
FROM dockcross/manylinux2010-x86
# Docker file for building gRPC manylinux-based Ruby artifacts.
# Updated: 2020-07-03
# install packages which rvm will require
RUN yum install -y autoconf gcc-c++ libtool readline-devel ruby sqlite-devel openssl-devel xz
# install rvm, RVM 1.26.0+ has signed releases, source rvm for usage outside of package scripts
RUN gpg --keyserver hkp://pool.sks-keyservers.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB && \
(curl -L http://get.rvm.io | sudo bash)
RUN echo "source /etc/profile.d/rvm.sh" >> /etc/rubybashrc
RUN bash -c " \
source /etc/rubybashrc && \
rvm cleanup all "
# Import patch files for ruby and gems
COPY build/patches /work/patches/
ENV BASH_ENV /etc/rubybashrc
# install rubies and fix permissions on
RUN bash -c " \
export CFLAGS='-s -O3 -fno-fast-math -fPIC' && \
for v in 2.5.7 ; do \
rvm install \$v --patch \$(echo /work/patches/ruby-\$v/* | tr ' ' ','); \
done && \
rvm cleanup all && \
find /usr/local/rvm -type d -print0 | sudo xargs -0 chmod g+sw "
# Install rake-compiler and typical gems in all Rubies
# do not generate documentation for gems
RUN echo "gem: --no-ri --no-rdoc" >> ~/.gemrc && \
bash -c " \
rvm all do gem update --system --no-document && \
rvm all do gem install --no-document bundler 'bundler:~>1.16' rake-compiler:1.1.0 hoe:3.20.0 mini_portile rubygems-tasks mini_portile2 && \
find /usr/local/rvm -type d -print0 | sudo xargs -0 chmod g+sw "
# Install rake-compiler's cross rubies in global dir instead of /root
RUN mkdir -p /usr/local/rake-compiler && \
ln -s /usr/local/rake-compiler ~/.rake-compiler
RUN bash -c " \
rvm alias create default 2.5.7 && \
rvm use default "
# Patch rake-compiler and hoe package
COPY build/patches2 /work/patches/
RUN cd /usr/local/rvm/gems/ruby-2.5.7/gems/rake-compiler-1.1.0 && \
( git apply /work/patches/rake-compiler-1.1.0/*.patch || true )
RUN cd /usr/local/rvm/gems/ruby-2.5.7/gems/hoe-3.20.0 && \
( git apply /work/patches/hoe-3.20.0/*.patch || true )
# Patch ruby-2.7.0 for cross build
RUN curl -SL http://cache.ruby-lang.org/pub/ruby/2.7/ruby-2.7.0.tar.xz | tar -xJC /root/ && \
cd /root/ruby-2.7.0 && \
git apply /work/patches/ruby-2.7.0/*.patch && \
cd .. && \
mkdir -p /usr/local/rake-compiler/sources/ && \
tar cjf /usr/local/rake-compiler/sources/ruby-2.7.0.tar.bz2 ruby-2.7.0 && \
rm -rf /root/ruby-2.7.0
ENV XRUBIES 2.7.0:2.6.0:2.5.0:2.4.0:2.3.0:2.2.2
# Build all xruby versions in parallel
# Then cleanup all build artifacts
RUN bash -c " \
export CFLAGS='-s -O1 -fno-omit-frame-pointer -fno-fast-math' && \
export MAKE='make V=0' && \
rake-compiler cross-ruby VERSION=$XRUBIES HOST=x86-linux-gnu && \
rm -rf ~/.rake-compiler/builds ~/.rake-compiler/sources && \
find /usr/local/rvm -type d -print0 | sudo xargs -0 chmod g+sw "
# Avoid linking against libruby shared object.
# See also https://github.com/rake-compiler/rake-compiler-dock/issues/13
RUN find /usr/local/rake-compiler/ruby/*linux*/ -name libruby.so | xargs rm
RUN find /usr/local/rake-compiler/ruby/*linux*/ -name libruby-static.a | while read f ; do cp $f `echo $f | sed s/-static//` ; done
RUN find /usr/local/rake-compiler/ruby/*linux*/ -name libruby.a | while read f ; do ar t $f | xargs ar d $f ; done
RUN find /usr/local/rake-compiler/ruby/*linux*/ -name mkmf.rb | while read f ; do sed -i ':a;N;$!ba;s/TRY_LINK = [^\n]*\n[^\n]*\n[^\n]*LOCAL_LIBS)/& -lruby-static/' $f ; done
# Fix paths in rake-compiler/config.yml and add rvm and mingw-tools to the global bashrc
RUN sed -i -- "s:/root/.rake-compiler:/usr/local/rake-compiler:g" /usr/local/rake-compiler/config.yml && \
echo "source /etc/profile.d/rvm.sh" >> /etc/bash.bashrc
# Install SIGINT forwarder
COPY build/sigfw.c /root/
RUN gcc $HOME/sigfw.c -o /usr/local/bin/sigfw
# Install patchelf_gem.sh
COPY build/patchelf_gem.sh /usr/local/bin/patchelf_gem.sh
ENV RUBY_CC_VERSION 2.7.0:2.6.0:2.5.0:2.4.0:2.3.0:2.2.2

@ -1 +1,91 @@
FROM larskanis/rake-compiler-dock-mri-x86_64-linux:1.0.0
FROM dockcross/manylinux2010-x64
# Docker file for building gRPC manylinux-based Ruby artifacts.
# Updated: 2020-07-03
# install packages which rvm will require
RUN yum install -y autoconf gcc-c++ libtool readline-devel ruby sqlite-devel openssl-devel xz
# install rvm, RVM 1.26.0+ has signed releases, source rvm for usage outside of package scripts
RUN gpg --keyserver hkp://pool.sks-keyservers.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB && \
(curl -L http://get.rvm.io | sudo bash)
RUN echo "source /etc/profile.d/rvm.sh" >> /etc/rubybashrc
RUN bash -c " \
source /etc/rubybashrc && \
rvm cleanup all "
# Import patch files for ruby and gems
COPY build/patches /work/patches/
ENV BASH_ENV /etc/rubybashrc
# install rubies and fix permissions on
RUN bash -c " \
export CFLAGS='-s -O3 -fno-fast-math -fPIC' && \
for v in 2.5.7 ; do \
rvm install \$v --patch \$(echo /work/patches/ruby-\$v/* | tr ' ' ','); \
done && \
rvm cleanup all && \
find /usr/local/rvm -type d -print0 | sudo xargs -0 chmod g+sw "
# Install rake-compiler and typical gems in all Rubies
# do not generate documentation for gems
RUN echo "gem: --no-ri --no-rdoc" >> ~/.gemrc && \
bash -c " \
rvm all do gem update --system --no-document && \
rvm all do gem install --no-document bundler 'bundler:~>1.16' rake-compiler:1.1.0 hoe:3.20.0 mini_portile rubygems-tasks mini_portile2 && \
find /usr/local/rvm -type d -print0 | sudo xargs -0 chmod g+sw "
# Install rake-compiler's cross rubies in global dir instead of /root
RUN mkdir -p /usr/local/rake-compiler && \
ln -s /usr/local/rake-compiler ~/.rake-compiler
RUN bash -c " \
rvm alias create default 2.5.7 && \
rvm use default "
# Patch rake-compiler and hoe package
COPY build/patches2 /work/patches/
RUN cd /usr/local/rvm/gems/ruby-2.5.7/gems/rake-compiler-1.1.0 && \
( git apply /work/patches/rake-compiler-1.1.0/*.patch || true )
RUN cd /usr/local/rvm/gems/ruby-2.5.7/gems/hoe-3.20.0 && \
( git apply /work/patches/hoe-3.20.0/*.patch || true )
# Patch ruby-2.7.0 for cross build
RUN curl -SL http://cache.ruby-lang.org/pub/ruby/2.7/ruby-2.7.0.tar.xz | tar -xJC /root/ && \
cd /root/ruby-2.7.0 && \
git apply /work/patches/ruby-2.7.0/*.patch && \
cd .. && \
mkdir -p /usr/local/rake-compiler/sources/ && \
tar cjf /usr/local/rake-compiler/sources/ruby-2.7.0.tar.bz2 ruby-2.7.0 && \
rm -rf /root/ruby-2.7.0
ENV XRUBIES 2.7.0:2.6.0:2.5.0:2.4.0:2.3.0:2.2.2
# Build all xruby versions in parallel
# Then cleanup all build artifacts
RUN bash -c " \
export CFLAGS='-s -O1 -fno-omit-frame-pointer -fno-fast-math' && \
export MAKE='make V=0' && \
rake-compiler cross-ruby VERSION=$XRUBIES HOST=x86_64-linux-gnu && \
rm -rf ~/.rake-compiler/builds ~/.rake-compiler/sources && \
find /usr/local/rvm -type d -print0 | sudo xargs -0 chmod g+sw "
# Avoid linking against libruby shared object.
# See also https://github.com/rake-compiler/rake-compiler-dock/issues/13
RUN find /usr/local/rake-compiler/ruby/*linux*/ -name libruby.so | xargs rm
RUN find /usr/local/rake-compiler/ruby/*linux*/ -name libruby-static.a | while read f ; do cp $f `echo $f | sed s/-static//` ; done
RUN find /usr/local/rake-compiler/ruby/*linux*/ -name libruby.a | while read f ; do ar t $f | xargs ar d $f ; done
RUN find /usr/local/rake-compiler/ruby/*linux*/ -name mkmf.rb | while read f ; do sed -i ':a;N;$!ba;s/TRY_LINK = [^\n]*\n[^\n]*\n[^\n]*LOCAL_LIBS)/& -lruby-static/' $f ; done
# Fix paths in rake-compiler/config.yml and add rvm and mingw-tools to the global bashrc
RUN sed -i -- "s:/root/.rake-compiler:/usr/local/rake-compiler:g" /usr/local/rake-compiler/config.yml && \
echo "source /etc/profile.d/rvm.sh" >> /etc/bash.bashrc
# Install SIGINT forwarder
COPY build/sigfw.c /root/
RUN gcc $HOME/sigfw.c -o /usr/local/bin/sigfw
# Install patchelf_gem.sh
COPY build/patchelf_gem.sh /usr/local/bin/patchelf_gem.sh
ENV RUBY_CC_VERSION 2.7.0:2.6.0:2.5.0:2.4.0:2.3.0:2.2.2

@ -30,8 +30,8 @@ end
def run_rake_compiler(platform, args)
require 'rake_compiler_dock'
ENV['RCD_RUBYVM'] = 'mri'
ENV['RCD_PLATFORM'] = platform
options = { :rubyvm => 'mri', :platform => platform }
options[:runas] = false if platform =~ /linux/
ENV['RCD_IMAGE'] = docker_image_for_rake_compiler(platform)
RakeCompilerDock.sh args
RakeCompilerDock.sh args, options
end

@ -29,7 +29,7 @@ cd -
DOCKERHUB_ORGANIZATION=grpctesting
for DOCKERFILE_DIR in tools/dockerfile/test/* tools/dockerfile/grpc_artifact_* tools/dockerfile/interoptest/* tools/dockerfile/distribtest/* third_party/rake-compiler-dock/*/
for DOCKERFILE_DIR in tools/dockerfile/test/* tools/dockerfile/grpc_artifact_* tools/dockerfile/interoptest/* tools/dockerfile/distribtest/*
do
# Generate image name based on Dockerfile checksum. That works well as long
# as can count on dockerfiles being written in a way that changing the logical

Loading…
Cancel
Save