diff --git a/tools/internal_ci/linux/grpc_distribtests_php.sh b/tools/internal_ci/linux/grpc_distribtests_php.sh index 969c93d04a4..9fd5ade49d7 100755 --- a/tools/internal_ci/linux/grpc_distribtests_php.sh +++ b/tools/internal_ci/linux/grpc_distribtests_php.sh @@ -30,8 +30,6 @@ tools/run_tests/task_runner.py -f artifact linux php ${TASK_RUNNER_EXTRA_FILTERS rm -rf input_artifacts mkdir -p input_artifacts cp -r artifacts/* input_artifacts/ || true -rm -rf artifacts_from_build_artifacts_step -mv artifacts artifacts_from_build_artifacts_step || true # This step simply collects php artifacts from subdirectories of input_artifacts/ and copies them to artifacts/ tools/run_tests/task_runner.py -f package linux php -x build_packages/sponge_log.xml || FAILED="true" diff --git a/tools/internal_ci/linux/grpc_distribtests_python.sh b/tools/internal_ci/linux/grpc_distribtests_python.sh index ddb01bab8f1..18671e1cea8 100755 --- a/tools/internal_ci/linux/grpc_distribtests_python.sh +++ b/tools/internal_ci/linux/grpc_distribtests_python.sh @@ -38,8 +38,6 @@ tools/run_tests/task_runner.py -f artifact linux python ${TASK_RUNNER_EXTRA_FILT rm -rf input_artifacts mkdir -p input_artifacts cp -r artifacts/* input_artifacts/ || true -rm -rf artifacts_from_build_artifacts_step -mv artifacts artifacts_from_build_artifacts_step || true # This step simply collects python artifacts from subdirectories of input_artifacts/ and copies them to artifacts/ tools/run_tests/task_runner.py -f package linux python -x build_packages/sponge_log.xml || FAILED="true" diff --git a/tools/internal_ci/linux/grpc_distribtests_ruby.sh b/tools/internal_ci/linux/grpc_distribtests_ruby.sh index 3d6ac6fcebe..8a658ed67fa 100755 --- a/tools/internal_ci/linux/grpc_distribtests_ruby.sh +++ b/tools/internal_ci/linux/grpc_distribtests_ruby.sh @@ -43,6 +43,9 @@ tools/run_tests/task_runner.py -f artifact linux ruby ${TASK_RUNNER_EXTRA_FILTER rm -rf input_artifacts mkdir -p input_artifacts cp -r artifacts/ruby_native_gem_*/* input_artifacts/ || true +# Also copy the gems directly to the "artifacts" directory, but do that without invoking ruby's "build_package" +# phase. +cp -r artifacts/ruby_native_gem_*/* artifacts/ || true # Run all ruby linux distribtests # We run the distribtests even if some of the artifacts have failed to build, since that gives diff --git a/tools/internal_ci/linux/release/grpc_collect_all_packages.cfg b/tools/internal_ci/linux/release/grpc_collect_all_packages.cfg new file mode 100644 index 00000000000..ad59fa9ceeb --- /dev/null +++ b/tools/internal_ci/linux/release/grpc_collect_all_packages.cfg @@ -0,0 +1,26 @@ +# Copyright 2022 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/release/grpc_collect_all_packages.sh" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} diff --git a/tools/internal_ci/linux/release/grpc_collect_all_packages.sh b/tools/internal_ci/linux/release/grpc_collect_all_packages.sh new file mode 100644 index 00000000000..2c3c5f8d4c8 --- /dev/null +++ b/tools/internal_ci/linux/release/grpc_collect_all_packages.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# Copyright 2022 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# avoid slow finalization after the script has exited. +source $(dirname $0)/../../../../tools/internal_ci/helper_scripts/move_src_tree_and_respawn_itself_rc + +# change to grpc repo root +cd $(dirname $0)/../../../.. + +source tools/internal_ci/helper_scripts/prepare_build_linux_rc + +# Move artifacts generated by the previous step in the build chain to a place +# where they can be accessed from within a docker container that builds +# the packages +mv ${KOKORO_GFILE_DIR}/github/grpc/artifacts input_artifacts || true +chmod +x input_artifacts/protoc*/* || true +ls -R input_artifacts || true + +# Collect the packages from the "build artifacts" phase and simply copy +# the files to the artifact/ directory. For some languages, +# artifacts from the "build artifacts" phase are actually combined +# into a single multi-platform package file: +# - C#'s Grpc.Tools package requires protoc plugins built on multiple platforms +# - Ruby's grpc.tools gem also requires protoc plugins build on multiple platforms +tools/run_tests/task_runner.py -f package linux -j 6 || FAILED="true" + +tools/internal_ci/helper_scripts/store_artifacts_from_moved_src_tree.sh + +if [ "$FAILED" != "" ] +then + exit 1 +fi + diff --git a/tools/internal_ci/linux/release/grpc_distribtests_csharp.cfg b/tools/internal_ci/linux/release/grpc_distribtests_csharp.cfg new file mode 100644 index 00000000000..e6abfb0b3c5 --- /dev/null +++ b/tools/internal_ci/linux/release/grpc_distribtests_csharp.cfg @@ -0,0 +1,26 @@ +# Copyright 2022 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_csharp.sh" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} diff --git a/tools/internal_ci/linux/release/grpc_distribtests_php.cfg b/tools/internal_ci/linux/release/grpc_distribtests_php.cfg new file mode 100644 index 00000000000..97dac40b06b --- /dev/null +++ b/tools/internal_ci/linux/release/grpc_distribtests_php.cfg @@ -0,0 +1,26 @@ +# Copyright 2022 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_php.sh" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} diff --git a/tools/internal_ci/linux/release/grpc_distribtests_python.cfg b/tools/internal_ci/linux/release/grpc_distribtests_python.cfg new file mode 100644 index 00000000000..6a916e93871 --- /dev/null +++ b/tools/internal_ci/linux/release/grpc_distribtests_python.cfg @@ -0,0 +1,26 @@ +# Copyright 2022 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_python.sh" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} diff --git a/tools/internal_ci/linux/release/grpc_distribtests_ruby.cfg b/tools/internal_ci/linux/release/grpc_distribtests_ruby.cfg new file mode 100644 index 00000000000..493dd497af2 --- /dev/null +++ b/tools/internal_ci/linux/release/grpc_distribtests_ruby.cfg @@ -0,0 +1,26 @@ +# Copyright 2022 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_ruby.sh" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} diff --git a/tools/internal_ci/macos/release/grpc_distribtests_csharp.cfg b/tools/internal_ci/macos/release/grpc_distribtests_csharp.cfg new file mode 100644 index 00000000000..0a84e7ce674 --- /dev/null +++ b/tools/internal_ci/macos/release/grpc_distribtests_csharp.cfg @@ -0,0 +1,27 @@ +# Copyright 2021 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/macos/grpc_distribtests_csharp.sh" +gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} diff --git a/tools/internal_ci/macos/release/grpc_distribtests_php.cfg b/tools/internal_ci/macos/release/grpc_distribtests_php.cfg new file mode 100644 index 00000000000..77b49b7fb50 --- /dev/null +++ b/tools/internal_ci/macos/release/grpc_distribtests_php.cfg @@ -0,0 +1,27 @@ +# Copyright 2022 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/macos/grpc_distribtests_php.sh" +gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} diff --git a/tools/internal_ci/macos/release/grpc_distribtests_python.cfg b/tools/internal_ci/macos/release/grpc_distribtests_python.cfg new file mode 100644 index 00000000000..e3d0b0c13d7 --- /dev/null +++ b/tools/internal_ci/macos/release/grpc_distribtests_python.cfg @@ -0,0 +1,27 @@ +# Copyright 2022 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/macos/grpc_distribtests_python.sh" +gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} diff --git a/tools/internal_ci/macos/release/grpc_distribtests_ruby.cfg b/tools/internal_ci/macos/release/grpc_distribtests_ruby.cfg new file mode 100644 index 00000000000..a4b6eb7182c --- /dev/null +++ b/tools/internal_ci/macos/release/grpc_distribtests_ruby.cfg @@ -0,0 +1,27 @@ +# Copyright 2022 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/macos/grpc_distribtests_ruby.sh" +gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} diff --git a/tools/internal_ci/windows/release/grpc_distribtests_csharp.cfg b/tools/internal_ci/windows/release/grpc_distribtests_csharp.cfg new file mode 100644 index 00000000000..aa26233d86c --- /dev/null +++ b/tools/internal_ci/windows/release/grpc_distribtests_csharp.cfg @@ -0,0 +1,26 @@ +# Copyright 2022 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/windows/grpc_distribtests_csharp.bat" +timeout_mins: 120 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} diff --git a/tools/internal_ci/windows/release/grpc_distribtests_python.cfg b/tools/internal_ci/windows/release/grpc_distribtests_python.cfg new file mode 100644 index 00000000000..85fd98f6ce6 --- /dev/null +++ b/tools/internal_ci/windows/release/grpc_distribtests_python.cfg @@ -0,0 +1,26 @@ +# Copyright 2022 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/windows/grpc_distribtests_python.bat" +timeout_mins: 120 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +}