diff --git a/tools/internal_ci/linux/grpc_distribtests_php.sh b/tools/internal_ci/linux/grpc_distribtests_php.sh index c2a5aa020da..969c93d04a4 100755 --- a/tools/internal_ci/linux/grpc_distribtests_php.sh +++ b/tools/internal_ci/linux/grpc_distribtests_php.sh @@ -33,7 +33,7 @@ cp -r artifacts/* input_artifacts/ || true rm -rf artifacts_from_build_artifacts_step mv artifacts artifacts_from_build_artifacts_step || true -# This step mostly just copies artifacts from input_artifacts (but it also does some wheel stripping) +# This step simply collects php artifacts from subdirectories of input_artifacts/ and copies them to artifacts/ tools/run_tests/task_runner.py -f package linux php -x build_packages/sponge_log.xml || FAILED="true" # the next step expects to find the artifacts from the previous step in the "input_artifacts" folder. diff --git a/tools/internal_ci/linux/grpc_distribtests_python.sh b/tools/internal_ci/linux/grpc_distribtests_python.sh index 2b6dae25263..ddb01bab8f1 100755 --- a/tools/internal_ci/linux/grpc_distribtests_python.sh +++ b/tools/internal_ci/linux/grpc_distribtests_python.sh @@ -41,7 +41,7 @@ cp -r artifacts/* input_artifacts/ || true rm -rf artifacts_from_build_artifacts_step mv artifacts artifacts_from_build_artifacts_step || true -# This step only copies artifacts from input_artifacts +# This step simply collects python artifacts from subdirectories of input_artifacts/ and copies them to artifacts/ tools/run_tests/task_runner.py -f package linux python -x build_packages/sponge_log.xml || FAILED="true" # the next step expects to find the artifacts from the previous step in the "input_artifacts" folder. diff --git a/tools/internal_ci/linux/grpc_distribtests_ruby.sh b/tools/internal_ci/linux/grpc_distribtests_ruby.sh index d91174430e7..3d6ac6fcebe 100755 --- a/tools/internal_ci/linux/grpc_distribtests_ruby.sh +++ b/tools/internal_ci/linux/grpc_distribtests_ruby.sh @@ -29,7 +29,7 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_ruby_artifact_rc # configure ccache source tools/internal_ci/helper_scripts/prepare_ccache_rc -# Build all ruby linux artifacts (this step actually builds all the binary wheels and source archives) +# Build all ruby linux artifacts (this step actually builds all the native and source gems) tools/run_tests/task_runner.py -f artifact linux ruby ${TASK_RUNNER_EXTRA_FILTERS} -j 6 --inner_jobs 6 -x build_artifacts/sponge_log.xml || FAILED="true" # Ruby "build_package" step is basically just a passthough for the "grpc" gems, so it's enough to just diff --git a/tools/internal_ci/macos/grpc_build_artifacts.sh b/tools/internal_ci/macos/grpc_build_artifacts.sh index b79ca418944..fc99c97539d 100755 --- a/tools/internal_ci/macos/grpc_build_artifacts.sh +++ b/tools/internal_ci/macos/grpc_build_artifacts.sh @@ -27,6 +27,7 @@ export PREPARE_BUILD_INSTALL_DEPS_RUBY=true export PREPARE_BUILD_INSTALL_DEPS_PHP=true source tools/internal_ci/helper_scripts/prepare_build_macos_rc +# TODO(jtattermusch): cleanup this prepare build step (needed for python artifact build) # install cython for all python versions python2.7 -m pip install -U cython setuptools==44.1.1 wheel --user python3.5 -m pip install -U cython setuptools==44.1.1 wheel --user diff --git a/tools/internal_ci/macos/grpc_distribtests_php.sh b/tools/internal_ci/macos/grpc_distribtests_php.sh index 5ae1ba1ab3c..7f2519e17c2 100644 --- a/tools/internal_ci/macos/grpc_distribtests_php.sh +++ b/tools/internal_ci/macos/grpc_distribtests_php.sh @@ -35,9 +35,13 @@ tools/run_tests/task_runner.py -f artifact macos php ${TASK_RUNNER_EXTRA_FILTERS # to upload its contents as job output artifacts rm -rf input_artifacts mkdir -p input_artifacts +# We could also copy the PHP artifact to artifacts/, but we intentionally don't do that +# in order to avoid hiding the PHP .tgz artifact built on linux (which has the same filename). +# The macos-built artifact will still show up in job's uploaded artifacts under the +# corresponding subdirectory ("php_pecl_package_macos_*") cp -r artifacts/php_pecl_package_macos_*/* input_artifacts/ || true -# Run all PHP linux distribtests +# Run all PHP macos distribtests # We run the distribtests even if some of the artifacts have failed to build, since that gives # a better signal about which distribtest are affected by the currently broken artifact builds. tools/run_tests/task_runner.py -f distribtest macos php ${TASK_RUNNER_EXTRA_FILTERS} -j 4 -x distribtests/sponge_log.xml || FAILED="true" diff --git a/tools/internal_ci/macos/grpc_distribtests_python.cfg b/tools/internal_ci/macos/grpc_distribtests_python.cfg new file mode 100644 index 00000000000..e3d0b0c13d7 --- /dev/null +++ b/tools/internal_ci/macos/grpc_distribtests_python.cfg @@ -0,0 +1,27 @@ +# Copyright 2022 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/macos/grpc_distribtests_python.sh" +gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} diff --git a/tools/internal_ci/macos/grpc_distribtests_python.sh b/tools/internal_ci/macos/grpc_distribtests_python.sh new file mode 100644 index 00000000000..5068bd19013 --- /dev/null +++ b/tools/internal_ci/macos/grpc_distribtests_python.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# Copyright 2022 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# avoid slow finalization after the script has exited. +source $(dirname $0)/../../../tools/internal_ci/helper_scripts/move_src_tree_and_respawn_itself_rc + +# change to grpc repo root +cd $(dirname $0)/../../.. + +export PREPARE_BUILD_INSTALL_DEPS_PYTHON=true +source tools/internal_ci/helper_scripts/prepare_build_macos_rc + +# TODO(jtattermusch): cleanup this prepare build step (needed for python artifact build) +# install cython for all python versions +python2.7 -m pip install -U cython setuptools==44.1.1 wheel --user +python3.5 -m pip install -U cython setuptools==44.1.1 wheel --user +python3.6 -m pip install -U cython setuptools==44.1.1 wheel --user +python3.7 -m pip install -U cython setuptools==44.1.1 wheel --user +python3.8 -m pip install -U cython setuptools==44.1.1 wheel --user +python3.9 -m pip install -U cython setuptools==44.1.1 wheel --user + +# Build all python macos artifacts (this step actually builds all the binary wheels and source archives) +tools/run_tests/task_runner.py -f artifact macos python ${TASK_RUNNER_EXTRA_FILTERS} -j 4 -x build_artifacts/sponge_log.xml || FAILED="true" + +# the next step expects to find the artifacts from the previous step in the "input_artifacts" folder. +rm -rf input_artifacts +mkdir -p input_artifacts +cp -r artifacts/* input_artifacts/ || true + +# Collect the python artifact from subdirectories of input_artifacts/ to artifacts/ +# TODO(jtattermusch): when collecting the artifacts that will later be uploaded as kokoro job artifacts, +# potentially skip some file names that would clash with linux-created artifacts. +cp -r input_artifacts/python_*/* artifacts/ || true + +# TODO(jtattermusch): Here we would normally run python macos distribtests, but currently no such tests are defined +# in distribtest_targets.py + +tools/internal_ci/helper_scripts/store_artifacts_from_moved_src_tree.sh + +if [ "$FAILED" != "" ] +then + exit 1 +fi diff --git a/tools/internal_ci/macos/grpc_distribtests_ruby.cfg b/tools/internal_ci/macos/grpc_distribtests_ruby.cfg new file mode 100644 index 00000000000..a4b6eb7182c --- /dev/null +++ b/tools/internal_ci/macos/grpc_distribtests_ruby.cfg @@ -0,0 +1,27 @@ +# Copyright 2022 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/macos/grpc_distribtests_ruby.sh" +gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} diff --git a/tools/internal_ci/macos/grpc_distribtests_ruby.sh b/tools/internal_ci/macos/grpc_distribtests_ruby.sh new file mode 100644 index 00000000000..411a7bce007 --- /dev/null +++ b/tools/internal_ci/macos/grpc_distribtests_ruby.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# Copyright 2022 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# avoid slow finalization after the script has exited. +source $(dirname $0)/../../../tools/internal_ci/helper_scripts/move_src_tree_and_respawn_itself_rc + +# change to grpc repo root +cd $(dirname $0)/../../.. + +export PREPARE_BUILD_INSTALL_DEPS_RUBY=true +source tools/internal_ci/helper_scripts/prepare_build_macos_rc + +# TODO(jtattermusch): can some of these steps be removed? +# needed to build ruby artifacts +gem install rubygems-update +update_rubygems +time bash tools/distrib/build_ruby_environment_macos.sh + +# Build all ruby macos artifacts (this step actually builds all the native and source gems) +tools/run_tests/task_runner.py -f artifact macos ruby ${TASK_RUNNER_EXTRA_FILTERS} -j 4 -x build_artifacts/sponge_log.xml || FAILED="true" + +# Ruby "build_package" step is basically just a passthough for the "grpc" gems, so it's enough to just +# copy the native gems directly to the "distribtests" step and skip the "build_package" phase entirely. +# Note that by skipping the "build_package" step, we are also skipping the build of "grpc-tools" gem +# but that's fine since the distribtests only test the "grpc" native gems. + +# The next step expects to find the artifacts from the previous step in the "input_artifacts" folder. +# in addition to that, preserve the contents of "artifacts" directory since we want kokoro +# to upload its contents as job output artifacts. +rm -rf input_artifacts +mkdir -p input_artifacts +cp -r artifacts/ruby_native_gem_*/* input_artifacts/ || true + +# TODO(jtattermusch): Here we would normally run ruby macos distribtests, but currently no such tests are defined +# in distribtest_targets.py + +tools/internal_ci/helper_scripts/store_artifacts_from_moved_src_tree.sh + +if [ "$FAILED" != "" ] +then + exit 1 +fi diff --git a/tools/internal_ci/macos/pull_request/grpc_distribtests_python.cfg b/tools/internal_ci/macos/pull_request/grpc_distribtests_python.cfg new file mode 100644 index 00000000000..4541d27ad5c --- /dev/null +++ b/tools/internal_ci/macos/pull_request/grpc_distribtests_python.cfg @@ -0,0 +1,32 @@ +# Copyright 2022 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/macos/grpc_distribtests_python.sh" +gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} + +env_vars { + key: "TASK_RUNNER_EXTRA_FILTERS" + value: "presubmit" +} diff --git a/tools/internal_ci/macos/pull_request/grpc_distribtests_ruby.cfg b/tools/internal_ci/macos/pull_request/grpc_distribtests_ruby.cfg new file mode 100644 index 00000000000..aa4557c7650 --- /dev/null +++ b/tools/internal_ci/macos/pull_request/grpc_distribtests_ruby.cfg @@ -0,0 +1,32 @@ +# Copyright 2022 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/macos/grpc_distribtests_ruby.sh" +gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} + +env_vars { + key: "TASK_RUNNER_EXTRA_FILTERS" + value: "presubmit" +}