[bazel] Reintroduce bazelified ruby and python tests (#34033)

Reintroduce https://github.com/grpc/grpc/pull/33959.

I added a fix for the python arm64 build (which is the reason why the
change has been reverted earlier).

templates/tools/dockerfile/passwordless_sudo.include

@@ -0,0 +1,6 @@
+# Passwordless sudo for all users
+# Bazel docker sandbox and bazel RBE run scripts under docker
+# as a regular user, but sometimes we need to be
+# able to do something with root privileges.
+RUN apt-get update && apt-get install -y sudo && apt-get clean
+RUN echo 'ALL ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers

templates/tools/dockerfile/ruby_deps.include

@@ -8,7 +8,8 @@ RUN \curl -sSL https://get.rvm.io | bash -s stable
 
 # Install Ruby 2.7
 RUN apt-get update && apt-get install -y procps && apt-get clean
-RUN /bin/bash -l -c "rvm install ruby-2.7"
+# "--disable-binary" is a workaround for https://github.com/rvm/rvm/issues/4975
+RUN /bin/bash -l -c "rvm install ruby-2.7 --disable-binary"
 RUN /bin/bash -l -c "rvm use --default ruby-2.7"
 RUN /bin/bash -l -c "echo 'gem: --no-document' > ~/.gemrc"
 RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc"

templates/tools/dockerfile/test/python_debian11_default_arm64/Dockerfile.template

@@ -25,7 +25,11 @@
   # for Python test coverage reporting
   RUN python3.9 -m pip install coverage
 
+  # six is required by the run_tests.py test harness
+  RUN python3.9 -m pip install six
+
   <%include file="../../gcp_api_libraries.include"/>
 
   <%include file="../../cmake.include"/>
   <%include file="../../ccache.include"/>
+  <%include file="../../git-jenkins.include"/>

templates/tools/dockerfile/test/python_debian11_default_x64/Dockerfile.template

@@ -31,6 +31,9 @@
   # for Python test coverage reporting
   RUN python3.9 -m pip install coverage
 
+  # six is required by the run_tests.py test harness
+  RUN python3.9 -m pip install six
+
   <%include file="../../gcp_api_libraries.include"/>
 
   <%include file="../../cmake.include"/>

templates/tools/dockerfile/test/ruby_debian11_x64/Dockerfile.template

@@ -23,6 +23,7 @@
   <%include file="../../ccache.include"/>
   <%include file="../../run_tests_addons.include"/>
   <%include file="../../xds_interop_deps.include"/>
+  <%include file="../../passwordless_sudo.include"/>
 
   # Define the default command.
   CMD ["bash"]

tools/bazelify_tests/dockerimage_current_versions.bzl

@@ -87,7 +87,7 @@ DOCKERIMAGE_CURRENT_VERSIONS = {
     "tools/dockerfile/interoptest/grpc_interop_php7.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_php7@sha256:09f4b895117c81506c423360b617917d06d3f7f0b78e4cca25eaec547ba6991e",
     "tools/dockerfile/interoptest/grpc_interop_python.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_python@sha256:fef1247f8256be2b9841331e7d21b0046da21a4a6d34a62addb36f62124725cf",
     "tools/dockerfile/interoptest/grpc_interop_pythonasyncio.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_pythonasyncio@sha256:bd4cdc8a71ef339193e178ce20d2b47a0b2aa25fc86c0b5740b9d86a2d4a0caa",
-    "tools/dockerfile/interoptest/grpc_interop_ruby.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_ruby@sha256:0de52450b29cf91365e623b020cd97722c307510ba1813bee09264e0a49acdbc",
+    "tools/dockerfile/interoptest/grpc_interop_ruby.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_ruby@sha256:596678dac1d736eae7fecd5fe718efcfd3dc637fe8adf626db86be7c7727b68b",
     "tools/dockerfile/interoptest/lb_interop_fake_servers.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/lb_interop_fake_servers@sha256:b89a51dd9147e1293f50ee64dd719fce5929ca7894d3770a3d80dbdecb99fd52",
     "tools/dockerfile/test/android_ndk.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/android_ndk@sha256:2bddf36ae504968b35f97e4a6c9b74864473689e84049675c30afb70f868d897",
     "tools/dockerfile/test/bazel.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/bazel@sha256:1118150d9d9479787165fff49f660a3dc633f1c57604305460172fc1916aa022",
@@ -108,10 +108,10 @@ DOCKERIMAGE_CURRENT_VERSIONS = {
     "tools/dockerfile/test/php7_debian11_arm64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/php7_debian11_arm64@sha256:444e25f9e3a89c2438e4d5e6f3904c5a1f4d1fb961f8456333ebe3e36e301a4e",
     "tools/dockerfile/test/php7_debian11_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/php7_debian11_x64@sha256:018d422abf144fc93e9027fd994f7d6aab453fffbe4a669d622dd3d1c1fe9ee7",
     "tools/dockerfile/test/python_alpine_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_alpine_x64@sha256:d10159225ae25276b7ae7bfc4230150e4b0a8ce7be833d904bdd4ecdfdc91c6e",
-    "tools/dockerfile/test/python_debian11_default_arm64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_debian11_default_arm64@sha256:868cfb50e465f086b75bb65a7fab6d15b1edefabcd8c1826340acefb6ea1737f",
+    "tools/dockerfile/test/python_debian11_default_arm64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_debian11_default_arm64@sha256:899d08818fa6d3311f473963b8c899e9036845418e184be18f30045ca996fb92",
-    "tools/dockerfile/test/python_debian11_default_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_debian11_default_x64@sha256:4f29e539941d22b7abb911f9b6b3101ff5c7c4fb75585bfe3b7389251ea6be1d",
+    "tools/dockerfile/test/python_debian11_default_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_debian11_default_x64@sha256:5e9278a10a8c338963fa258213910fc0dead6bac91110256c1a0a344e0ade58f",
     "tools/dockerfile/test/rbe_ubuntu2004.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/rbe_ubuntu2004@sha256:d3951aeadf43e3bee6adc5b86d26cdaf0b9d1b5baf790d7b2530d1c197adc9f8",
-    "tools/dockerfile/test/ruby_debian11_arm64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_arm64@sha256:9503d80a40555aba4dd531b64354ad8036c6b37e162c93e7994ca23d89bc7d41",
+    "tools/dockerfile/test/ruby_debian11_arm64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_arm64@sha256:c4901beb737a6aed3969c7bc601cd441488e5283a6abfabb80210c1bd2f5cd19",
-    "tools/dockerfile/test/ruby_debian11_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_x64@sha256:3f01369c3e5707fa63007820b30461b9e32a4b729d81cb92d19669d7966a8584",
+    "tools/dockerfile/test/ruby_debian11_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_x64@sha256:f30272c98928898b42903aa1a08fb0d769c8dc7b6173231a778c8c001b904219",
     "tools/dockerfile/test/sanity.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/sanity@sha256:189e07d8503aa15344e3c8f565783659c3e2edc5b8ca455ec427de1e29ef4504",
 }

tools/bazelify_tests/test/BUILD

@@ -59,29 +59,28 @@ grpc_run_tests_harness_test(
     docker_image_version = "tools/dockerfile/test/cxx_debian11_x64.current_version",
 )
 
-# TODO(jtattermusch): Reintroduce ruby tests once they pass.
+# Ruby
-# # Ruby
+grpc_run_tests_harness_test(
-# grpc_run_tests_harness_test(
+    name = "runtests_ruby_linux_dbg",
-#     name = "runtests_ruby_linux_dbg",
+    size = "enormous",
-#     size = "enormous",
+    args = [
-#     args = [
+        "-l ruby -c dbg",
-#         "-l ruby -c dbg",
+    ],
-#     ],
+    docker_image_version = "tools/dockerfile/test/ruby_debian11_x64.current_version",
-#     docker_image_version = "tools/dockerfile/test/ruby_debian11_x64.current_version",
+    prepare_script = ":prepare_ruby.sh",
-#     prepare_script = ":prepare_ruby.sh",
+    use_login_shell = True,  # ruby's docker image uses RVM which wierdly requires login shell
-#     use_login_shell = True,  # ruby's docker image uses RVM which wierdly requires login shell
+)
-# )
 
-# grpc_run_tests_harness_test(
+grpc_run_tests_harness_test(
-#     name = "runtests_ruby_linux_opt",
+    name = "runtests_ruby_linux_opt",
-#     size = "enormous",
+    size = "enormous",
-#     args = [
+    args = [
-#         "-l ruby -c opt",
+        "-l ruby -c opt",
-#     ],
+    ],
-#     docker_image_version = "tools/dockerfile/test/ruby_debian11_x64.current_version",
+    docker_image_version = "tools/dockerfile/test/ruby_debian11_x64.current_version",
-#     prepare_script = ":prepare_ruby.sh",
+    prepare_script = ":prepare_ruby.sh",
-#     use_login_shell = True,  # ruby's docker image uses RVM which wierdly requires login shell
+    use_login_shell = True,  # ruby's docker image uses RVM which wierdly requires login shell
-# )
+)
 
 # PHP
 grpc_run_tests_harness_test(
@@ -102,16 +101,15 @@ grpc_run_tests_harness_test(
     docker_image_version = "tools/dockerfile/test/php7_debian11_x64.current_version",
 )
 
-# TODO(jtattermusch): Reintroduce python tests once they pass.
+# Python
-# # Python
+grpc_run_tests_harness_test(
-# grpc_run_tests_harness_test(
+    name = "runtests_python_linux_opt",
-#     name = "runtests_python_linux_opt",
+    size = "enormous",
-#     size = "enormous",
+    args = [
-#     args = [
+        "-l python -c opt",
-#         "-l python -c opt",
+    ],
-#     ],
+    docker_image_version = "tools/dockerfile/test/python_debian11_default_x64.current_version",
-#     docker_image_version = "tools/dockerfile/test/python_debian11_default_x64.current_version",
+)
-# )
 
 # C#
 grpc_run_tests_harness_test(
@@ -144,9 +142,9 @@ test_suite(
         # TODO(jtattermusch): reenable the test once not flaky anymore
         #":runtests_php_linux_dbg",
         ":runtests_php_linux_opt",
-        #":runtests_python_linux_opt",
+        ":runtests_python_linux_opt",
-        #":runtests_ruby_linux_dbg",
+        ":runtests_ruby_linux_dbg",
-        #":runtests_ruby_linux_opt",
+        ":runtests_ruby_linux_opt",
     ],
 )
 

tools/bazelify_tests/test/portability_tests.bzl

@@ -73,17 +73,16 @@ def generate_run_tests_portability_tests(name):
             )
             test_names.append(test_name)
 
-    # TODO(jtattermusch): Reintroduce the test once it passes.
     # Python on alpine
-    #grpc_run_tests_harness_test(
+    grpc_run_tests_harness_test(
-    #    name = "runtests_python_linux_dbg_alpine",
+        name = "runtests_python_linux_dbg_alpine",
-    #    args = [
+        args = [
-    #        "-l python -c dbg --compiler python_alpine",
+            "-l python -c dbg --compiler python_alpine",
-    #    ],
+        ],
-    #    docker_image_version = "tools/dockerfile/test/python_alpine_x64.current_version",
+        docker_image_version = "tools/dockerfile/test/python_alpine_x64.current_version",
-    #    size = "enormous",
+        size = "enormous",
-    #)
+    )
-    #test_names.append("runtests_python_linux_dbg_alpine")
+    test_names.append("runtests_python_linux_dbg_alpine")
 
     # Generate test suite that allows easily running all portability tests.
     native.test_suite(

tools/bazelify_tests/test/prepare_ruby.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+# Copyright 2023 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# make sure /usr/local/rvm is writable by ruby when running under docker as non-root
+sudo chown -R "$(id -u)" /usr/local/rvm
+

tools/dockerfile/interoptest/grpc_interop_ruby.current_version

@@ -1 +1 @@
-us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_ruby:4abe304d586e1bc862cb85e9ac5e9c7b2f7ae73c@sha256:0de52450b29cf91365e623b020cd97722c307510ba1813bee09264e0a49acdbc
+us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_ruby:a6f7413e822f23ea4b696afe916714ba3df5175c@sha256:596678dac1d736eae7fecd5fe718efcfd3dc637fe8adf626db86be7c7727b68b

tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile

@@ -88,7 +88,8 @@ RUN \curl -sSL https://get.rvm.io | bash -s stable
 
 # Install Ruby 2.7
 RUN apt-get update && apt-get install -y procps && apt-get clean
-RUN /bin/bash -l -c "rvm install ruby-2.7"
+# "--disable-binary" is a workaround for https://github.com/rvm/rvm/issues/4975
+RUN /bin/bash -l -c "rvm install ruby-2.7 --disable-binary"
 RUN /bin/bash -l -c "rvm use --default ruby-2.7"
 RUN /bin/bash -l -c "echo 'gem: --no-document' > ~/.gemrc"
 RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc"

tools/dockerfile/test/python_debian11_default_arm64.current_version

@@ -1 +1 @@
-us-docker.pkg.dev/grpc-testing/testing-images-public/python_debian11_default_arm64:faff3d19982d4a3f479a4d9633d28028834bc4df@sha256:868cfb50e465f086b75bb65a7fab6d15b1edefabcd8c1826340acefb6ea1737f
+us-docker.pkg.dev/grpc-testing/testing-images-public/python_debian11_default_arm64:3d13ca7bdbe30730bd150966e4b6f8e476b6b63e@sha256:899d08818fa6d3311f473963b8c899e9036845418e184be18f30045ca996fb92

tools/dockerfile/test/python_debian11_default_arm64/Dockerfile

@@ -63,6 +63,9 @@ RUN apt-get update && apt-get install -y python3.9 python3.9-dev python3-pip
 # for Python test coverage reporting
 RUN python3.9 -m pip install coverage
 
+# six is required by the run_tests.py test harness
+RUN python3.9 -m pip install six
+
 # Google Cloud Platform API libraries
 # These are needed for uploading test results to BigQuery (e.g. by tools/run_tests scripts)
 RUN python3 -m pip install --upgrade google-auth==1.23.0 google-api-python-client==1.12.8 oauth2client==4.1.0
@@ -88,3 +91,9 @@ RUN curl -sSL -o ccache.tar.gz https://github.com/ccache/ccache/releases/downloa
     && cd ../.. \
     && rm -rf ccache-4.5.1 ccache.tar.gz
 
+#=================
+# Setup git to access working directory across docker boundary
+
+RUN git config --global --add safe.directory /var/local/jenkins/grpc
+RUN git config --global protocol.file.allow always
+

tools/dockerfile/test/python_debian11_default_x64.current_version

@@ -1 +1 @@
-us-docker.pkg.dev/grpc-testing/testing-images-public/python_debian11_default_x64:31ceb004af0498642b186d28e5c6411b974c2c04@sha256:4f29e539941d22b7abb911f9b6b3101ff5c7c4fb75585bfe3b7389251ea6be1d
+us-docker.pkg.dev/grpc-testing/testing-images-public/python_debian11_default_x64:14e1a18ec5d6bb304cfed659d4b4d5360b13b52c@sha256:5e9278a10a8c338963fa258213910fc0dead6bac91110256c1a0a344e0ade58f

tools/dockerfile/test/python_debian11_default_x64/Dockerfile

@@ -170,6 +170,9 @@ RUN apt-get update && apt-get install -y python3.9 python3.9-dev python3-pip
 # for Python test coverage reporting
 RUN python3.9 -m pip install coverage
 
+# six is required by the run_tests.py test harness
+RUN python3.9 -m pip install six
+
 # Google Cloud Platform API libraries
 # These are needed for uploading test results to BigQuery (e.g. by tools/run_tests scripts)
 RUN python3 -m pip install --upgrade google-auth==1.23.0 google-api-python-client==1.12.8 oauth2client==4.1.0

tools/dockerfile/test/ruby_debian11_arm64.current_version

@@ -1 +1 @@
-us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_arm64:215df896c707f55c3092a64afc4075c7a408b516@sha256:9503d80a40555aba4dd531b64354ad8036c6b37e162c93e7994ca23d89bc7d41
+us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_arm64:cb472b623590186bc1d6e996cccb6fbb094e7e3a@sha256:c4901beb737a6aed3969c7bc601cd441488e5283a6abfabb80210c1bd2f5cd19

tools/dockerfile/test/ruby_debian11_arm64/Dockerfile

@@ -88,7 +88,8 @@ RUN \curl -sSL https://get.rvm.io | bash -s stable
 
 # Install Ruby 2.7
 RUN apt-get update && apt-get install -y procps && apt-get clean
-RUN /bin/bash -l -c "rvm install ruby-2.7"
+# "--disable-binary" is a workaround for https://github.com/rvm/rvm/issues/4975
+RUN /bin/bash -l -c "rvm install ruby-2.7 --disable-binary"
 RUN /bin/bash -l -c "rvm use --default ruby-2.7"
 RUN /bin/bash -l -c "echo 'gem: --no-document' > ~/.gemrc"
 RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc"

tools/dockerfile/test/ruby_debian11_x64.current_version

@@ -1 +1 @@
-us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_x64:05ff6c04182addf087f77b2515992e98b9c7a171@sha256:3f01369c3e5707fa63007820b30461b9e32a4b729d81cb92d19669d7966a8584
+us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_x64:b697ef647022cc3a14350ded1e01a52dfd9b8cf3@sha256:f30272c98928898b42903aa1a08fb0d769c8dc7b6173231a778c8c001b904219

tools/dockerfile/test/ruby_debian11_x64/Dockerfile

@@ -88,7 +88,8 @@ RUN \curl -sSL https://get.rvm.io | bash -s stable
 
 # Install Ruby 2.7
 RUN apt-get update && apt-get install -y procps && apt-get clean
-RUN /bin/bash -l -c "rvm install ruby-2.7"
+# "--disable-binary" is a workaround for https://github.com/rvm/rvm/issues/4975
+RUN /bin/bash -l -c "rvm install ruby-2.7 --disable-binary"
 RUN /bin/bash -l -c "rvm use --default ruby-2.7"
 RUN /bin/bash -l -c "echo 'gem: --no-document' > ~/.gemrc"
 RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc"
@@ -121,6 +122,13 @@ RUN mkdir /var/local/jenkins
 # Required by XDS interop test harness.
 RUN python3 -m pip install virtualenv==16.7.9
 
+# Passwordless sudo for all users
+# Bazel docker sandbox and bazel RBE run scripts under docker
+# as a regular user, but sometimes we need to be
+# able to do something with root privileges.
+RUN apt-get update && apt-get install -y sudo && apt-get clean
+RUN echo 'ALL ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
 
 # Define the default command.
 CMD ["bash"]

tools/run_tests/helper_scripts/build_ruby.sh

@@ -45,4 +45,4 @@ popd
 
 # unbreak subsequent make builds by restoring zconf.h (previously renamed by cmake build)
 # see https://github.com/madler/zlib/issues/133
-(cd third_party/zlib; git checkout zconf.h)
+(cd third_party/zlib; git checkout zconf.h || cp zconf.h.included zconf.h)