Cherry-pick to get Ruby 3.1 support in protobuf 3.20.0 (#9657)

* Allow pre-compiled binaries for ruby 3.1.0 (#9566)

* Allow pre-compiled binaries for ruby 3.1.1

* add comment

* fix build and use ruby 3.1.0

* add ruby31 to build CI for tests and release

* trying to fix ci

* install ruby 3.1.0 in ruby_build_environment.sh

* use head for rvm to install 3.1.0

* just install master version of rvm in prepare_build_macos_rc

* force install of master rvm in ruby_build_environment.sh

* Use coroutine=universal when compiling ruby31

* use ucontext

* fix filename

* fix coroutine name

* use git head for rake-compiler-dock

* use newest rake-compiler-dock version

* Updated CHANGES.txt for Ruby changes.

* Fixed Ruby 3.1 tests by marking intersect? as unimplemented. (#9645)

* Fixed Ruby 3.1 tests by marking intersect? as unimplemented.

* Updated compatibility tests.

Co-authored-by: Marco Concetto Rudilosso <marcoconcettorudilosso@gmail.com>

CHANGES.txt

@@ -2,8 +2,11 @@
 
   Ruby
   * Dropped Ruby 2.3 and 2.4 support for CI and releases. (#9311)
-  * Message.decode/encode: Add max_recursion_depth option (#9218)
+  * Added Ruby 3.1 support for CI and releases (#9566).
-  * Rename max_recursion_depth to recursion_limit (#9486)
+  * Message.decode/encode: Add recursion_limit option (#9218/#9486)
+  * Allocate with xrealloc()/xfree() so message allocation is visible to the
+    Ruby GC.  In certain tests this leads to much lower memory usage due to more
+    frequent GC runs (#9586).
   * Fix conversion of singleton classes in Ruby (#9342)
   * Suppress warning for intentional circular require (#9556)
   * JSON will now output shorter strings for double and float fields when possible

kokoro/linux/dockerfile/test/ruby/Dockerfile

@@ -35,6 +35,7 @@ RUN /bin/bash -l -c "rvm install 2.5.1"
 RUN /bin/bash -l -c "rvm install 2.6.0"
 RUN /bin/bash -l -c "rvm install 2.7.0"
 RUN /bin/bash -l -c "rvm install 3.0.0"
+RUN /bin/bash -l -c "rvm install 3.1.0"
 RUN /bin/bash -l -c "rvm install jruby-9.2.20.1"
 RUN /bin/bash -l -c "rvm install jruby-9.3.3.0"
 

kokoro/linux/ruby31/build.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+#
+# This is the top-level script we give to Kokoro as the entry point for
+# running the "pull request" project:
+#
+# This script selects a specific Dockerfile (for building a Docker image) and
+# a script to run inside that image.  Then we delegate to the general
+# build_and_run_docker.sh script.
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+export DOCKERHUB_ORGANIZATION=protobuftesting
+export DOCKERFILE_DIR=kokoro/linux/dockerfile/test/ruby
+export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh
+export OUTPUT_DIR=testoutput
+export TEST_SET="ruby31"
+./kokoro/linux/build_and_run_docker.sh

kokoro/linux/ruby31/continuous.cfg

@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/ruby31/build.sh"
+timeout_mins: 120
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.xml"
+  }
+}

kokoro/linux/ruby31/presubmit.cfg

@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/ruby31/build.sh"
+timeout_mins: 120
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.xml"
+  }
+}

kokoro/macos/prepare_build_macos_rc

@@ -36,5 +36,5 @@ if [[ "${KOKORO_INSTALL_RVM:-}" == "yes" ]] ; then
   # Old OpenSSL versions cannot handle the SSL certificate used by
   # https://get.rvm.io, so as a workaround we download RVM directly from
   # GitHub. See this issue for details: https://github.com/rvm/rvm/issues/5133
-  curl -sSL https://raw.githubusercontent.com/rvm/rvm/master/binscripts/rvm-installer | bash -s stable --ruby
+  curl -sSL https://raw.githubusercontent.com/rvm/rvm/master/binscripts/rvm-installer | bash -s master --ruby
 fi

kokoro/macos/ruby31/build.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+#
+# Build file to set up and run tests
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+# Prepare worker environment to run tests
+KOKORO_INSTALL_RVM=yes
+source kokoro/macos/prepare_build_macos_rc
+
+./tests.sh ruby31

kokoro/macos/ruby31/continuous.cfg

@@ -0,0 +1,5 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/macos/ruby31/build.sh"
+timeout_mins: 1440

kokoro/macos/ruby31/presubmit.cfg

@@ -0,0 +1,5 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/macos/ruby31/build.sh"
+timeout_mins: 1440

kokoro/release/ruby/macos/ruby/ruby_build_environment.sh

@@ -3,12 +3,25 @@
 set -ex
 
 set +ex  # rvm script is very verbose and exits with errorcode
+
+curl -sSL https://rvm.io/mpapis.asc | gpg --import -
+curl -sSL https://rvm.io/pkuczynski.asc | gpg --import -
+
+# Old OpenSSL versions cannot handle the SSL certificate used by
+# https://get.rvm.io, so as a workaround we download RVM directly from
+# GitHub. See this issue for details: https://github.com/rvm/rvm/issues/5133
+curl -sSL https://raw.githubusercontent.com/rvm/rvm/master/binscripts/rvm-installer | bash -s master --ruby
+
 source $HOME/.rvm/scripts/rvm
 set -e  # rvm commands are very verbose
 time rvm install 2.5.0
 rvm use 2.5.0
 gem install rake-compiler --no-document
 gem install bundler --no-document
+time rvm install 3.1.0
+rvm use 3.1.0
+gem install rake-compiler --no-document
+gem install bundler --no-document
 time rvm install 2.7.0
 rvm use 2.7.0 --default
 gem install rake-compiler --no-document
@@ -20,6 +33,8 @@ set -ex
 rm -rf ~/.rake-compiler
 
 CROSS_RUBY=$(mktemp tmpfile.XXXXXXXX)
+CROSS_RUBY31=$(mktemp tmpfile.XXXXXXXX)
+
 
 curl https://raw.githubusercontent.com/rake-compiler/rake-compiler/72184e51779b6a3b9b8580b036a052fdc3181ced/tasks/bin/cross-ruby.rake > "$CROSS_RUBY"
 
@@ -52,8 +67,33 @@ patch "$CROSS_RUBY" << EOF
  end
 EOF
 
+cp $CROSS_RUBY $CROSS_RUBY31
+
+patch "$CROSS_RUBY31" << EOF
+--- cross-ruby.rake	2022-03-04 11:49:52.000000000 +0000
++++ patched	2022-03-04 11:58:22.000000000 +0000
+@@ -114,6 +114,7 @@
+     '--enable-static',
+     '--disable-shared',
+     '--disable-install-doc',
++    '--with-coroutine=ucontext',
+     '--without-gmp',
+     '--with-ext=',
+     'LDFLAGS=-pipe',
+EOF
+
 MAKE="make -j8"
 
+set +x # rvm commands are very verbose
+rvm use 3.1.0
+set -x
+ruby --version | grep 'ruby 3.1.0'
+for v in 3.1.0 ; do
+  ccache -c
+  rake -f "$CROSS_RUBY31" cross-ruby VERSION="$v" HOST=x86_64-darwin MAKE="$MAKE"
+  rake -f "$CROSS_RUBY31" cross-ruby VERSION="$v" HOST=aarch64-darwin MAKE="$MAKE"
+done
+
 set +x # rvm commands are very verbose
 rvm use 2.7.0
 set -x

ruby/Rakefile

@@ -127,7 +127,7 @@ else
     ['x86-mingw32', 'x64-mingw32', 'x86_64-linux', 'x86-linux'].each do |plat|
       RakeCompilerDock.sh <<-"EOT", platform: plat
         bundle && \
-        IN_DOCKER=true rake native:#{plat} pkg/#{spec.full_name}-#{plat}.gem RUBY_CC_VERSION=3.0.0:2.7.0:2.6.0:2.5.0
+        IN_DOCKER=true rake native:#{plat} pkg/#{spec.full_name}-#{plat}.gem RUBY_CC_VERSION=3.1.0:3.0.0:2.7.0:2.6.0:2.5.0
       EOT
     end
   end
@@ -135,7 +135,7 @@ else
   if RUBY_PLATFORM =~ /darwin/
     task 'gem:native' do
       system "rake genproto"
-      system "rake cross native gem RUBY_CC_VERSION=3.0.0:2.7.0:2.6.0:2.5.1"
+      system "rake cross native gem RUBY_CC_VERSION=3.1.0:3.0.0:2.7.0:2.6.0:2.5.1"
     end
   else
     task 'gem:native' => [:genproto, 'gem:windows', 'gem:java']

ruby/compatibility_tests/v3.0.0/tests/repeated_field_test.rb

@@ -21,6 +21,7 @@ class RepeatedFieldTest < Test::Unit::TestCase
       :nitems, :iter_for_reverse_each, :indexes, :append, :prepend]
     arr_methods -= [:union, :difference, :filter!]
     arr_methods -= [:intersection, :deconstruct] # ruby 2.7 methods we can ignore
+    arr_methods -= [:intersect?] # ruby 3.1 methods we can ignore
     arr_methods.each do |method_name|
       assert m.repeated_string.respond_to?(method_name) == true, "does not respond to #{method_name}"
     end

ruby/google-protobuf.gemspec

@@ -17,8 +17,7 @@ Gem::Specification.new do |s|
   else
     s.files     += Dir.glob('ext/**/*')
     s.extensions= ["ext/google/protobuf_c/extconf.rb"]
-    s.add_development_dependency "rake-compiler-dock", "= 1.1.0"
+    s.add_development_dependency "rake-compiler-dock", "= 1.2.1"  end
-  end
   s.test_files  = ["tests/basic.rb",
                   "tests/stress.rb",
                   "tests/generated_code_test.rb"]

ruby/tests/repeated_field_test.rb

@@ -21,6 +21,7 @@ class RepeatedFieldTest < Test::Unit::TestCase
       :nitems, :iter_for_reverse_each, :indexes, :append, :prepend]
     arr_methods -= [:union, :difference, :filter!]
     arr_methods -= [:intersection, :deconstruct] # ruby 2.7 methods we can ignore
+    arr_methods -= [:intersect?] # ruby 3.1 methods we can ignore
     arr_methods.each do |method_name|
       assert m.repeated_string.respond_to?(method_name) == true, "does not respond to #{method_name}"
     end

tests.sh

@@ -422,6 +422,10 @@ build_ruby30() {
   internal_build_cpp  # For conformance tests.
   cd ruby && bash travis-test.sh ruby-3.0.2 && cd ..
 }
+build_ruby31() {
+  internal_build_cpp  # For conformance tests.
+  cd ruby && bash travis-test.sh ruby-3.1.0 && cd ..
+}
 
 build_jruby92() {
   internal_build_cpp                # For conformance tests.
@@ -593,6 +597,7 @@ Usage: $0 { cpp |
             ruby26 |
             ruby27 |
             ruby30 |
+            ruby31 |
             jruby92 |
             jruby93 |
             ruby_all |