Add macos grpc_distribtests_python and grpc_distribtests_ruby (#29141)

* small cleanup in singlejob distribtest jobs

* add macos grpc_distribtest_ruby and grpc_distribtest_python jobs

tools/internal_ci/linux/grpc_distribtests_php.sh

@@ -33,7 +33,7 @@ cp -r artifacts/* input_artifacts/ || true
 rm -rf artifacts_from_build_artifacts_step
 mv artifacts artifacts_from_build_artifacts_step || true
 
-# This step mostly just copies artifacts from input_artifacts (but it also does some wheel stripping)
+# This step simply collects php artifacts from subdirectories of input_artifacts/ and copies them to artifacts/
 tools/run_tests/task_runner.py -f package linux php -x build_packages/sponge_log.xml || FAILED="true"
 
 # the next step expects to find the artifacts from the previous step in the "input_artifacts" folder.

tools/internal_ci/linux/grpc_distribtests_python.sh

@@ -41,7 +41,7 @@ cp -r artifacts/* input_artifacts/ || true
 rm -rf artifacts_from_build_artifacts_step
 mv artifacts artifacts_from_build_artifacts_step || true
 
-# This step only copies artifacts from input_artifacts
+# This step simply collects python artifacts from subdirectories of input_artifacts/ and copies them to artifacts/
 tools/run_tests/task_runner.py -f package linux python -x build_packages/sponge_log.xml || FAILED="true"
 
 # the next step expects to find the artifacts from the previous step in the "input_artifacts" folder.

tools/internal_ci/linux/grpc_distribtests_ruby.sh

@@ -29,7 +29,7 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_ruby_artifact_rc
 # configure ccache
 source tools/internal_ci/helper_scripts/prepare_ccache_rc
 
-# Build all ruby linux artifacts (this step actually builds all the binary wheels and source archives)
+# Build all ruby linux artifacts (this step actually builds all the native and source gems)
 tools/run_tests/task_runner.py -f artifact linux ruby ${TASK_RUNNER_EXTRA_FILTERS} -j 6 --inner_jobs 6 -x build_artifacts/sponge_log.xml || FAILED="true"
 
 # Ruby "build_package" step is basically just a passthough for the "grpc" gems, so it's enough to just

tools/internal_ci/macos/grpc_build_artifacts.sh

@@ -27,6 +27,7 @@ export PREPARE_BUILD_INSTALL_DEPS_RUBY=true
 export PREPARE_BUILD_INSTALL_DEPS_PHP=true
 source tools/internal_ci/helper_scripts/prepare_build_macos_rc
 
+# TODO(jtattermusch): cleanup this prepare build step (needed for python artifact build)
 # install cython for all python versions
 python2.7 -m pip install -U cython setuptools==44.1.1 wheel --user
 python3.5 -m pip install -U cython setuptools==44.1.1 wheel --user

tools/internal_ci/macos/grpc_distribtests_php.sh

@@ -35,9 +35,13 @@ tools/run_tests/task_runner.py -f artifact macos php ${TASK_RUNNER_EXTRA_FILTERS
 # to upload its contents as job output artifacts
 rm -rf input_artifacts
 mkdir -p input_artifacts
+# We could also copy the PHP artifact to artifacts/, but we intentionally don't do that
+# in order to avoid hiding the PHP .tgz artifact built on linux (which has the same filename).
+# The macos-built artifact will still show up in job's uploaded artifacts under the
+# corresponding subdirectory ("php_pecl_package_macos_*")
 cp -r artifacts/php_pecl_package_macos_*/* input_artifacts/ || true
 
-# Run all PHP linux distribtests
+# Run all PHP macos distribtests
 # We run the distribtests even if some of the artifacts have failed to build, since that gives
 # a better signal about which distribtest are affected by the currently broken artifact builds.
 tools/run_tests/task_runner.py -f distribtest macos php ${TASK_RUNNER_EXTRA_FILTERS} -j 4 -x distribtests/sponge_log.xml || FAILED="true"

tools/internal_ci/macos/grpc_distribtests_python.cfg

@@ -0,0 +1,27 @@
+# Copyright 2022 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/macos/grpc_distribtests_python.sh"
+gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json"
+timeout_mins: 240
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.*"
+    regex: "github/grpc/reports/**"
+    regex: "github/grpc/artifacts/**"
+  }
+}

tools/internal_ci/macos/grpc_distribtests_python.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+# Copyright 2022 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# avoid slow finalization after the script has exited.
+source $(dirname $0)/../../../tools/internal_ci/helper_scripts/move_src_tree_and_respawn_itself_rc
+
+# change to grpc repo root
+cd $(dirname $0)/../../..
+
+export PREPARE_BUILD_INSTALL_DEPS_PYTHON=true
+source tools/internal_ci/helper_scripts/prepare_build_macos_rc
+
+# TODO(jtattermusch): cleanup this prepare build step (needed for python artifact build)
+# install cython for all python versions
+python2.7 -m pip install -U cython setuptools==44.1.1 wheel --user
+python3.5 -m pip install -U cython setuptools==44.1.1 wheel --user
+python3.6 -m pip install -U cython setuptools==44.1.1 wheel --user
+python3.7 -m pip install -U cython setuptools==44.1.1 wheel --user
+python3.8 -m pip install -U cython setuptools==44.1.1 wheel --user
+python3.9 -m pip install -U cython setuptools==44.1.1 wheel --user
+
+# Build all python macos artifacts (this step actually builds all the binary wheels and source archives)
+tools/run_tests/task_runner.py -f artifact macos python ${TASK_RUNNER_EXTRA_FILTERS} -j 4 -x build_artifacts/sponge_log.xml || FAILED="true"
+
+# the next step expects to find the artifacts from the previous step in the "input_artifacts" folder.
+rm -rf input_artifacts
+mkdir -p input_artifacts
+cp -r artifacts/* input_artifacts/ || true
+
+# Collect the python artifact from subdirectories of input_artifacts/ to artifacts/
+# TODO(jtattermusch): when collecting the artifacts that will later be uploaded as kokoro job artifacts,
+# potentially skip some file names that would clash with linux-created artifacts.
+cp -r input_artifacts/python_*/* artifacts/ || true
+
+# TODO(jtattermusch): Here we would normally run python macos distribtests, but currently no such tests are defined
+# in distribtest_targets.py
+
+tools/internal_ci/helper_scripts/store_artifacts_from_moved_src_tree.sh
+
+if [ "$FAILED" != "" ]
+then
+  exit 1
+fi

tools/internal_ci/macos/grpc_distribtests_ruby.cfg

@@ -0,0 +1,27 @@
+# Copyright 2022 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/macos/grpc_distribtests_ruby.sh"
+gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json"
+timeout_mins: 240
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.*"
+    regex: "github/grpc/reports/**"
+    regex: "github/grpc/artifacts/**"
+  }
+}

tools/internal_ci/macos/grpc_distribtests_ruby.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+# Copyright 2022 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# avoid slow finalization after the script has exited.
+source $(dirname $0)/../../../tools/internal_ci/helper_scripts/move_src_tree_and_respawn_itself_rc
+
+# change to grpc repo root
+cd $(dirname $0)/../../..
+
+export PREPARE_BUILD_INSTALL_DEPS_RUBY=true
+source tools/internal_ci/helper_scripts/prepare_build_macos_rc
+
+# TODO(jtattermusch): can some of these steps be removed?
+# needed to build ruby artifacts
+gem install rubygems-update
+update_rubygems
+time bash tools/distrib/build_ruby_environment_macos.sh
+
+# Build all ruby macos artifacts (this step actually builds all the native and source gems)
+tools/run_tests/task_runner.py -f artifact macos ruby ${TASK_RUNNER_EXTRA_FILTERS} -j 4 -x build_artifacts/sponge_log.xml || FAILED="true"
+
+# Ruby "build_package" step is basically just a passthough for the "grpc" gems, so it's enough to just
+# copy the native gems directly to the "distribtests" step and skip the "build_package" phase entirely.
+# Note that by skipping the "build_package" step, we are also skipping the build of "grpc-tools" gem
+# but that's fine since the distribtests only test the "grpc" native gems.
+
+# The next step expects to find the artifacts from the previous step in the "input_artifacts" folder.
+# in addition to that, preserve the contents of "artifacts" directory since we want kokoro
+# to upload its contents as job output artifacts.
+rm -rf input_artifacts
+mkdir -p input_artifacts
+cp -r artifacts/ruby_native_gem_*/* input_artifacts/ || true
+
+# TODO(jtattermusch): Here we would normally run ruby macos distribtests, but currently no such tests are defined
+# in distribtest_targets.py
+
+tools/internal_ci/helper_scripts/store_artifacts_from_moved_src_tree.sh
+
+if [ "$FAILED" != "" ]
+then
+  exit 1
+fi

tools/internal_ci/macos/pull_request/grpc_distribtests_python.cfg

@@ -0,0 +1,32 @@
+# Copyright 2022 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/macos/grpc_distribtests_python.sh"
+gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json"
+timeout_mins: 240
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.*"
+    regex: "github/grpc/reports/**"
+    regex: "github/grpc/artifacts/**"
+  }
+}
+
+env_vars {
+  key: "TASK_RUNNER_EXTRA_FILTERS"
+  value: "presubmit"
+}

tools/internal_ci/macos/pull_request/grpc_distribtests_ruby.cfg

@@ -0,0 +1,32 @@
+# Copyright 2022 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/macos/grpc_distribtests_ruby.sh"
+gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json"
+timeout_mins: 240
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.*"
+    regex: "github/grpc/reports/**"
+    regex: "github/grpc/artifacts/**"
+  }
+}
+
+env_vars {
+  key: "TASK_RUNNER_EXTRA_FILTERS"
+  value: "presubmit"
+}