add streamlined and faster ruby distribtest (#27124)

tools/internal_ci/linux/grpc_distribtests_ruby.cfg

@@ -0,0 +1,26 @@
+# Copyright 2021 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_ruby.sh"
+timeout_mins: 240
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.*"
+    regex: "github/grpc/reports/**"
+    regex: "github/grpc/artifacts/**"
+  }
+}

tools/internal_ci/linux/grpc_distribtests_ruby.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+# Copyright 2021 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# change to grpc repo root
+cd $(dirname $0)/../../..
+
+source tools/internal_ci/helper_scripts/prepare_build_linux_rc
+
+set +ex
+[[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh
+set -e  # rvm commands are very verbose
+rvm --default use ruby-2.4.1
+set -ex
+
+# Build all ruby linux artifacts (this step actually builds all the binary wheels and source archives)
+tools/run_tests/task_runner.py -f artifact linux ruby -j 6 -x build_artifacts/sponge_log.xml || FAILED="true"
+
+# Ruby "build_package" step is basically just a passthough for the "grpc" gems, so it's enough to just
+# copy the native gems directly to the "distribtests" step and skip the "build_package" phase entirely.
+# Note that by skipping the "build_package" step, we are also skipping the build of "grpc-tools" gem
+# but that's fine since the distribtests only test the "grpc" native gems.
+
+# The next step expects to find the artifacts from the previous step in the "input_artifacts" folder.
+# in addition to that, preserve the contents of "artifacts" directory since we want kokoro
+# to upload its contents as job output artifacts.
+rm -rf input_artifacts
+mkdir -p input_artifacts
+cp -r artifacts/ruby_native_gem_*/* input_artifacts/ || true
+
+# Run all ruby linux distribtests
+# We run the distribtests even if some of the artifacts have failed to build, since that gives
+# a better signal about which distribtest are affected by the currently broken artifact builds.
+tools/run_tests/task_runner.py -f distribtest linux ruby -j 6 -x distribtests/sponge_log.xml || FAILED="true"
+
+if [ "$FAILED" != "" ]
+then
+  exit 1
+fi

tools/run_tests/artifacts/build_package_ruby.sh

@@ -21,16 +21,18 @@ base=$(pwd)
 
 mkdir -p artifacts/
 
-# All the ruby packages have been built in the artifact phase already
+# All the "grpc" gems have been built in the artifact phase already
 # and we only collect them here to deliver them to the distribtest phase.
+# NOTE: Besides the platform-specific native gems, all the artifact build
+# jobs will generate a grpc-X.Y.Z.gem source package, and only one of them
+# will end up in the artifacts/ directory. They should be all equivalent
+# though.
 cp -r "${EXTERNAL_GIT_ROOT}"/input_artifacts/ruby_native_gem_*/* artifacts/ || true
 
+# Next, build the "grpc-tools" gem by collecting the protoc and grpc_ruby_plugin binaries
+# that have been built by the the artifact build phase previously.
 well_known_protos=( any api compiler/plugin descriptor duration empty field_mask source_context struct timestamp type wrappers )
 
-# TODO: all the artifact builder configurations generate a grpc-VERSION.gem
-# source distribution package, and only one of them will end up
-# in the artifacts/ directory. They should be all equivalent though.
-
 for arch in {x86,x64}; do
   case $arch in
     x64)