diff --git a/tools/internal_ci/linux/grpc_build_artifacts.sh b/tools/internal_ci/linux/grpc_build_artifacts.sh index 0810790c37b..130c3492198 100755 --- a/tools/internal_ci/linux/grpc_build_artifacts.sh +++ b/tools/internal_ci/linux/grpc_build_artifacts.sh @@ -29,7 +29,7 @@ set -e # rvm commands are very verbose rvm --default use ruby-2.4.1 set -ex -tools/run_tests/task_runner.py -f artifact linux -j 12 || FAILED="true" +tools/run_tests/task_runner.py -f artifact linux ${TASK_RUNNER_EXTRA_FILTERS} -j 12 || FAILED="true" tools/internal_ci/helper_scripts/store_artifacts_from_moved_src_tree.sh diff --git a/tools/internal_ci/linux/grpc_distribtests_php.sh b/tools/internal_ci/linux/grpc_distribtests_php.sh index aad8d65b9bb..c2a5aa020da 100755 --- a/tools/internal_ci/linux/grpc_distribtests_php.sh +++ b/tools/internal_ci/linux/grpc_distribtests_php.sh @@ -24,7 +24,7 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_rc # Build all PHP linux artifacts -tools/run_tests/task_runner.py -f artifact linux php -j 4 -x build_artifacts/sponge_log.xml || FAILED="true" +tools/run_tests/task_runner.py -f artifact linux php ${TASK_RUNNER_EXTRA_FILTERS} -j 4 -x build_artifacts/sponge_log.xml || FAILED="true" # the next step expects to find the artifacts from the previous step in the "input_artifacts" folder. rm -rf input_artifacts @@ -46,7 +46,7 @@ cp -r artifacts/* input_artifacts/ || true # Run all PHP linux distribtests # We run the distribtests even if some of the artifacts have failed to build, since that gives # a better signal about which distribtest are affected by the currently broken artifact builds. -tools/run_tests/task_runner.py -f distribtest linux php -j 4 -x distribtests/sponge_log.xml || FAILED="true" +tools/run_tests/task_runner.py -f distribtest linux php ${TASK_RUNNER_EXTRA_FILTERS} -j 4 -x distribtests/sponge_log.xml || FAILED="true" tools/internal_ci/helper_scripts/store_artifacts_from_moved_src_tree.sh diff --git a/tools/internal_ci/linux/grpc_distribtests_python.sh b/tools/internal_ci/linux/grpc_distribtests_python.sh index a03309231d5..874e978dfb0 100755 --- a/tools/internal_ci/linux/grpc_distribtests_python.sh +++ b/tools/internal_ci/linux/grpc_distribtests_python.sh @@ -29,7 +29,7 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_rc source tools/internal_ci/helper_scripts/prepare_qemu_rc # Build all python linux artifacts (this step actually builds all the binary wheels and source archives) -tools/run_tests/task_runner.py -f artifact linux python -j 12 -x build_artifacts/sponge_log.xml || FAILED="true" +tools/run_tests/task_runner.py -f artifact linux python ${TASK_RUNNER_EXTRA_FILTERS} -j 12 -x build_artifacts/sponge_log.xml || FAILED="true" # the next step expects to find the artifacts from the previous step in the "input_artifacts" folder. rm -rf input_artifacts @@ -51,7 +51,7 @@ cp -r artifacts/* input_artifacts/ || true # Run all python linux distribtests # We run the distribtests even if some of the artifacts have failed to build, since that gives # a better signal about which distribtest are affected by the currently broken artifact builds. -tools/run_tests/task_runner.py -f distribtest linux python -j 12 -x distribtests/sponge_log.xml || FAILED="true" +tools/run_tests/task_runner.py -f distribtest linux python ${TASK_RUNNER_EXTRA_FILTERS} -j 12 -x distribtests/sponge_log.xml || FAILED="true" tools/internal_ci/helper_scripts/store_artifacts_from_moved_src_tree.sh diff --git a/tools/internal_ci/linux/grpc_distribtests_ruby.sh b/tools/internal_ci/linux/grpc_distribtests_ruby.sh index 3da15534b8a..0d848617781 100755 --- a/tools/internal_ci/linux/grpc_distribtests_ruby.sh +++ b/tools/internal_ci/linux/grpc_distribtests_ruby.sh @@ -30,7 +30,7 @@ rvm --default use ruby-2.4.1 set -ex # Build all ruby linux artifacts (this step actually builds all the binary wheels and source archives) -tools/run_tests/task_runner.py -f artifact linux ruby -j 6 -x build_artifacts/sponge_log.xml || FAILED="true" +tools/run_tests/task_runner.py -f artifact linux ruby ${TASK_RUNNER_EXTRA_FILTERS} -j 6 -x build_artifacts/sponge_log.xml || FAILED="true" # Ruby "build_package" step is basically just a passthough for the "grpc" gems, so it's enough to just # copy the native gems directly to the "distribtests" step and skip the "build_package" phase entirely. @@ -47,7 +47,7 @@ cp -r artifacts/ruby_native_gem_*/* input_artifacts/ || true # Run all ruby linux distribtests # We run the distribtests even if some of the artifacts have failed to build, since that gives # a better signal about which distribtest are affected by the currently broken artifact builds. -tools/run_tests/task_runner.py -f distribtest linux ruby -j 6 -x distribtests/sponge_log.xml || FAILED="true" +tools/run_tests/task_runner.py -f distribtest linux ruby ${TASK_RUNNER_EXTRA_FILTERS} -j 6 -x distribtests/sponge_log.xml || FAILED="true" tools/internal_ci/helper_scripts/store_artifacts_from_moved_src_tree.sh diff --git a/tools/internal_ci/linux/grpc_distribtests_standalone.sh b/tools/internal_ci/linux/grpc_distribtests_standalone.sh index f71f3b13773..dd854400c98 100755 --- a/tools/internal_ci/linux/grpc_distribtests_standalone.sh +++ b/tools/internal_ci/linux/grpc_distribtests_standalone.sh @@ -23,4 +23,4 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_rc -tools/run_tests/task_runner.py -f distribtest linux cpp -j 6 +tools/run_tests/task_runner.py -f distribtest linux cpp ${TASK_RUNNER_EXTRA_FILTERS} -j 6 diff --git a/tools/internal_ci/linux/pull_request/grpc_build_artifacts.cfg b/tools/internal_ci/linux/pull_request/grpc_build_artifacts.cfg new file mode 100644 index 00000000000..8fcc5e77567 --- /dev/null +++ b/tools/internal_ci/linux/pull_request/grpc_build_artifacts.cfg @@ -0,0 +1,31 @@ +# Copyright 2021 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_build_artifacts.sh" +timeout_mins: 180 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} + +env_vars { + key: "TASK_RUNNER_EXTRA_FILTERS" + value: "presubmit" +} diff --git a/tools/internal_ci/linux/pull_request/grpc_distribtests_php.cfg b/tools/internal_ci/linux/pull_request/grpc_distribtests_php.cfg new file mode 100644 index 00000000000..cc5f4f74e95 --- /dev/null +++ b/tools/internal_ci/linux/pull_request/grpc_distribtests_php.cfg @@ -0,0 +1,31 @@ +# Copyright 2021 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_php.sh" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} + +env_vars { + key: "TASK_RUNNER_EXTRA_FILTERS" + value: "presubmit" +} diff --git a/tools/internal_ci/linux/pull_request/grpc_distribtests_python.cfg b/tools/internal_ci/linux/pull_request/grpc_distribtests_python.cfg new file mode 100644 index 00000000000..de62ae1c80b --- /dev/null +++ b/tools/internal_ci/linux/pull_request/grpc_distribtests_python.cfg @@ -0,0 +1,31 @@ +# Copyright 2021 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_python.sh" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} + +env_vars { + key: "TASK_RUNNER_EXTRA_FILTERS" + value: "presubmit" +} diff --git a/tools/internal_ci/linux/pull_request/grpc_distribtests_ruby.cfg b/tools/internal_ci/linux/pull_request/grpc_distribtests_ruby.cfg new file mode 100644 index 00000000000..3963c5b5287 --- /dev/null +++ b/tools/internal_ci/linux/pull_request/grpc_distribtests_ruby.cfg @@ -0,0 +1,31 @@ +# Copyright 2021 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_ruby.sh" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} + +env_vars { + key: "TASK_RUNNER_EXTRA_FILTERS" + value: "presubmit" +} diff --git a/tools/internal_ci/linux/pull_request/grpc_distribtests_standalone.cfg b/tools/internal_ci/linux/pull_request/grpc_distribtests_standalone.cfg new file mode 100644 index 00000000000..b7e27a97841 --- /dev/null +++ b/tools/internal_ci/linux/pull_request/grpc_distribtests_standalone.cfg @@ -0,0 +1,31 @@ +# Copyright 2021 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_standalone.sh" +timeout_mins: 120 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} + +env_vars { + key: "TASK_RUNNER_EXTRA_FILTERS" + value: "presubmit" +} diff --git a/tools/internal_ci/macos/grpc_build_artifacts.sh b/tools/internal_ci/macos/grpc_build_artifacts.sh index 2e4522a6f91..e6941bd14fe 100755 --- a/tools/internal_ci/macos/grpc_build_artifacts.sh +++ b/tools/internal_ci/macos/grpc_build_artifacts.sh @@ -41,7 +41,7 @@ time bash tools/distrib/build_ruby_environment_macos.sh gem install rubygems-update update_rubygems -tools/run_tests/task_runner.py -f artifact macos || FAILED="true" +tools/run_tests/task_runner.py -f artifact macos ${TASK_RUNNER_EXTRA_FILTERS} || FAILED="true" tools/internal_ci/helper_scripts/store_artifacts_from_moved_src_tree.sh diff --git a/tools/internal_ci/macos/grpc_distribtests_php.sh b/tools/internal_ci/macos/grpc_distribtests_php.sh index 5bd716c32fc..5ae1ba1ab3c 100644 --- a/tools/internal_ci/macos/grpc_distribtests_php.sh +++ b/tools/internal_ci/macos/grpc_distribtests_php.sh @@ -25,7 +25,7 @@ export PREPARE_BUILD_INSTALL_DEPS_PHP=true source tools/internal_ci/helper_scripts/prepare_build_macos_rc # Build all PHP macos artifacts -tools/run_tests/task_runner.py -f artifact macos php -j 4 -x build_artifacts/sponge_log.xml || FAILED="true" +tools/run_tests/task_runner.py -f artifact macos php ${TASK_RUNNER_EXTRA_FILTERS} -j 4 -x build_artifacts/sponge_log.xml || FAILED="true" # PHP's "build_package" step is basically just a passthough, so the build artifacts can be used # directly by the "distribtests" step (and we skip the "build_package" phase entirely on mac). @@ -40,7 +40,7 @@ cp -r artifacts/php_pecl_package_macos_*/* input_artifacts/ || true # Run all PHP linux distribtests # We run the distribtests even if some of the artifacts have failed to build, since that gives # a better signal about which distribtest are affected by the currently broken artifact builds. -tools/run_tests/task_runner.py -f distribtest macos php -j 4 -x distribtests/sponge_log.xml || FAILED="true" +tools/run_tests/task_runner.py -f distribtest macos php ${TASK_RUNNER_EXTRA_FILTERS} -j 4 -x distribtests/sponge_log.xml || FAILED="true" tools/internal_ci/helper_scripts/store_artifacts_from_moved_src_tree.sh diff --git a/tools/internal_ci/macos/pull_request/grpc_build_artifacts.cfg b/tools/internal_ci/macos/pull_request/grpc_build_artifacts.cfg new file mode 100644 index 00000000000..ee61cd24a4f --- /dev/null +++ b/tools/internal_ci/macos/pull_request/grpc_build_artifacts.cfg @@ -0,0 +1,32 @@ +# Copyright 2021 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/macos/grpc_build_artifacts.sh" +gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} + +env_vars { + key: "TASK_RUNNER_EXTRA_FILTERS" + value: "presubmit" +} diff --git a/tools/internal_ci/macos/pull_request/grpc_distribtests_php.cfg b/tools/internal_ci/macos/pull_request/grpc_distribtests_php.cfg new file mode 100644 index 00000000000..192ab8271d7 --- /dev/null +++ b/tools/internal_ci/macos/pull_request/grpc_distribtests_php.cfg @@ -0,0 +1,32 @@ +# Copyright 2021 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/macos/grpc_distribtests_php.sh" +gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} + +env_vars { + key: "TASK_RUNNER_EXTRA_FILTERS" + value: "presubmit" +} diff --git a/tools/internal_ci/windows/grpc_build_artifacts.bat b/tools/internal_ci/windows/grpc_build_artifacts.bat index 72813bd4b59..10c52cfd9fa 100644 --- a/tools/internal_ci/windows/grpc_build_artifacts.bat +++ b/tools/internal_ci/windows/grpc_build_artifacts.bat @@ -29,7 +29,7 @@ cd /d %~dp0\..\..\.. set PREPARE_BUILD_INSTALL_DEPS_PYTHON=true call tools/internal_ci/helper_scripts/prepare_build_windows.bat || exit /b 1 -python tools/run_tests/task_runner.py -f artifact windows -j 4 +python tools/run_tests/task_runner.py -f artifact windows %TASK_RUNNER_EXTRA_FILTERS% -j 4 set RUNTESTS_EXITCODE=%errorlevel% bash tools/internal_ci/helper_scripts/store_artifacts_from_moved_src_tree.sh diff --git a/tools/internal_ci/windows/grpc_distribtests_standalone.bat b/tools/internal_ci/windows/grpc_distribtests_standalone.bat index 1d0635723f3..184c7b808f9 100644 --- a/tools/internal_ci/windows/grpc_distribtests_standalone.bat +++ b/tools/internal_ci/windows/grpc_distribtests_standalone.bat @@ -24,7 +24,7 @@ cd /d %~dp0\..\..\.. call tools/internal_ci/helper_scripts/prepare_build_windows.bat || exit /b 1 -python tools/run_tests/task_runner.py -f distribtest windows cpp -j 4 +python tools/run_tests/task_runner.py -f distribtest windows cpp %TASK_RUNNER_EXTRA_FILTERS% -j 4 set RUNTESTS_EXITCODE=%errorlevel% exit /b %RUNTESTS_EXITCODE% diff --git a/tools/internal_ci/windows/pull_request/grpc_build_artifacts.cfg b/tools/internal_ci/windows/pull_request/grpc_build_artifacts.cfg new file mode 100644 index 00000000000..244437a268b --- /dev/null +++ b/tools/internal_ci/windows/pull_request/grpc_build_artifacts.cfg @@ -0,0 +1,31 @@ +# Copyright 2021 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/windows/grpc_build_artifacts.bat" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} + +env_vars { + key: "TASK_RUNNER_EXTRA_FILTERS" + value: "presubmit" +} diff --git a/tools/internal_ci/windows/pull_request/grpc_distribtests_standalone.cfg b/tools/internal_ci/windows/pull_request/grpc_distribtests_standalone.cfg new file mode 100644 index 00000000000..b673e13ba11 --- /dev/null +++ b/tools/internal_ci/windows/pull_request/grpc_distribtests_standalone.cfg @@ -0,0 +1,31 @@ +# Copyright 2021 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/windows/grpc_distribtests_standalone.bat" +timeout_mins: 120 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} + +env_vars { + key: "TASK_RUNNER_EXTRA_FILTERS" + value: "presubmit" +} diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index 1932b46f0ce..86755f2ed95 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -99,11 +99,13 @@ _ARCH_FLAG_MAP = {'x86': '-m32', 'x64': '-m64'} class PythonArtifact: """Builds Python artifacts.""" - def __init__(self, platform, arch, py_version): + def __init__(self, platform, arch, py_version, presubmit=False): self.name = 'python_%s_%s_%s' % (platform, arch, py_version) self.platform = platform self.arch = arch self.labels = ['artifact', 'python', platform, arch, py_version] + if presubmit: + self.labels.append('presubmit') self.py_version = py_version if 'manylinux' in platform: self.labels.append('linux') @@ -191,11 +193,13 @@ class PythonArtifact: class RubyArtifact: """Builds ruby native gem.""" - def __init__(self, platform, arch): + def __init__(self, platform, arch, presubmit=False): self.name = 'ruby_native_gem_%s_%s' % (platform, arch) self.platform = platform self.arch = arch self.labels = ['artifact', 'ruby', platform, arch] + if presubmit: + self.labels.append('presubmit') def pre_build_jobspecs(self): return [] @@ -212,7 +216,7 @@ class RubyArtifact: class CSharpExtArtifact: """Builds C# native extension library""" - def __init__(self, platform, arch, arch_abi=None): + def __init__(self, platform, arch, arch_abi=None, presubmit=False): self.name = 'csharp_ext_%s_%s' % (platform, arch) self.platform = platform self.arch = arch @@ -221,6 +225,8 @@ class CSharpExtArtifact: if arch_abi: self.name += '_%s' % arch_abi self.labels.append(arch_abi) + if presubmit: + self.labels.append('presubmit') def pre_build_jobspecs(self): return [] @@ -270,11 +276,13 @@ class CSharpExtArtifact: class PHPArtifact: """Builds PHP PECL package""" - def __init__(self, platform, arch): + def __init__(self, platform, arch, presubmit=False): self.name = 'php_pecl_package_{0}_{1}'.format(platform, arch) self.platform = platform self.arch = arch self.labels = ['artifact', 'php', platform, arch] + if presubmit: + self.labels.append('presubmit') def pre_build_jobspecs(self): return [] @@ -294,11 +302,13 @@ class PHPArtifact: class ProtocArtifact: """Builds protoc and protoc-plugin artifacts""" - def __init__(self, platform, arch): + def __init__(self, platform, arch, presubmit=False): self.name = 'protoc_%s_%s' % (platform, arch) self.platform = platform self.arch = arch self.labels = ['artifact', 'protoc', platform, arch] + if presubmit: + self.labels.append('presubmit') def pre_build_jobspecs(self): return [] @@ -343,66 +353,73 @@ class ProtocArtifact: def targets(): """Gets list of supported targets""" return [ - ProtocArtifact('linux', 'x64'), - ProtocArtifact('linux', 'x86'), - ProtocArtifact('linux', 'aarch64'), - ProtocArtifact('macos', 'x64'), - ProtocArtifact('windows', 'x64'), - ProtocArtifact('windows', 'x86'), - CSharpExtArtifact('linux', 'x64'), - CSharpExtArtifact('linux', 'aarch64'), - CSharpExtArtifact('macos', 'x64'), - CSharpExtArtifact('windows', 'x64'), - CSharpExtArtifact('windows', 'x86'), - CSharpExtArtifact('linux', 'android', arch_abi='arm64-v8a'), - CSharpExtArtifact('linux', 'android', arch_abi='armeabi-v7a'), - CSharpExtArtifact('linux', 'android', arch_abi='x86'), - CSharpExtArtifact('macos', 'ios'), - PythonArtifact('manylinux2014', 'x64', 'cp36-cp36m'), + ProtocArtifact('linux', 'x64', presubmit=True), + ProtocArtifact('linux', 'x86', presubmit=True), + ProtocArtifact('linux', 'aarch64', presubmit=True), + ProtocArtifact('macos', 'x64', presubmit=True), + ProtocArtifact('windows', 'x64', presubmit=True), + ProtocArtifact('windows', 'x86', presubmit=True), + CSharpExtArtifact('linux', 'x64', presubmit=True), + CSharpExtArtifact('linux', 'aarch64', presubmit=True), + CSharpExtArtifact('macos', 'x64', presubmit=True), + CSharpExtArtifact('windows', 'x64', presubmit=True), + CSharpExtArtifact('windows', 'x86', presubmit=True), + CSharpExtArtifact('linux', + 'android', + arch_abi='arm64-v8a', + presubmit=True), + CSharpExtArtifact('linux', + 'android', + arch_abi='armeabi-v7a', + presubmit=True), + CSharpExtArtifact('linux', 'android', arch_abi='x86', presubmit=True), + CSharpExtArtifact('macos', 'ios', presubmit=True), + PythonArtifact('manylinux2014', 'x64', 'cp36-cp36m', presubmit=True), PythonArtifact('manylinux2014', 'x64', 'cp37-cp37m'), PythonArtifact('manylinux2014', 'x64', 'cp38-cp38'), PythonArtifact('manylinux2014', 'x64', 'cp39-cp39'), - PythonArtifact('manylinux2014', 'x64', 'cp310-cp310'), - PythonArtifact('manylinux2014', 'x86', 'cp36-cp36m'), + PythonArtifact('manylinux2014', 'x64', 'cp310-cp310', presubmit=True), + PythonArtifact('manylinux2014', 'x86', 'cp36-cp36m', presubmit=True), PythonArtifact('manylinux2014', 'x86', 'cp37-cp37m'), PythonArtifact('manylinux2014', 'x86', 'cp38-cp38'), PythonArtifact('manylinux2014', 'x86', 'cp39-cp39'), - PythonArtifact('manylinux2014', 'x86', 'cp310-cp310'), + PythonArtifact('manylinux2014', 'x86', 'cp310-cp310', presubmit=True), PythonArtifact('manylinux2010', 'x64', 'cp36-cp36m'), - PythonArtifact('manylinux2010', 'x64', 'cp37-cp37m'), + PythonArtifact('manylinux2010', 'x64', 'cp37-cp37m', presubmit=True), PythonArtifact('manylinux2010', 'x64', 'cp38-cp38'), PythonArtifact('manylinux2010', 'x64', 'cp39-cp39'), PythonArtifact('manylinux2010', 'x86', 'cp36-cp36m'), - PythonArtifact('manylinux2010', 'x86', 'cp37-cp37m'), + PythonArtifact('manylinux2010', 'x86', 'cp37-cp37m', presubmit=True), PythonArtifact('manylinux2010', 'x86', 'cp38-cp38'), PythonArtifact('manylinux2010', 'x86', 'cp39-cp39'), - PythonArtifact('manylinux2014', 'aarch64', 'cp36-cp36m'), + PythonArtifact('manylinux2014', 'aarch64', 'cp36-cp36m', + presubmit=True), PythonArtifact('manylinux2014', 'aarch64', 'cp37-cp37m'), - PythonArtifact('manylinux2014', 'aarch64', 'cp38-cp38'), + PythonArtifact('manylinux2014', 'aarch64', 'cp38-cp38', presubmit=True), PythonArtifact('manylinux2014', 'aarch64', 'cp39-cp39'), PythonArtifact('manylinux2014', 'aarch64', 'cp310-cp310'), - PythonArtifact('linux_extra', 'armv7', 'cp36-cp36m'), + PythonArtifact('linux_extra', 'armv7', 'cp36-cp36m', presubmit=True), PythonArtifact('linux_extra', 'armv7', 'cp37-cp37m'), PythonArtifact('linux_extra', 'armv7', 'cp38-cp38'), PythonArtifact('linux_extra', 'armv7', 'cp39-cp39'), - PythonArtifact('linux_extra', 'armv7', 'cp310-cp310'), - PythonArtifact('macos', 'x64', 'python3.6'), + PythonArtifact('linux_extra', 'armv7', 'cp310-cp310', presubmit=True), + PythonArtifact('macos', 'x64', 'python3.6', presubmit=True), PythonArtifact('macos', 'x64', 'python3.7'), PythonArtifact('macos', 'x64', 'python3.8'), PythonArtifact('macos', 'x64', 'python3.9'), - PythonArtifact('macos', 'x64', 'python3.10'), - PythonArtifact('windows', 'x86', 'Python36_32bit'), + PythonArtifact('macos', 'x64', 'python3.10', presubmit=True), + PythonArtifact('windows', 'x86', 'Python36_32bit', presubmit=True), PythonArtifact('windows', 'x86', 'Python37_32bit'), PythonArtifact('windows', 'x86', 'Python38_32bit'), PythonArtifact('windows', 'x86', 'Python39_32bit'), - PythonArtifact('windows', 'x86', 'Python310_32bit'), - PythonArtifact('windows', 'x64', 'Python36'), + PythonArtifact('windows', 'x86', 'Python310_32bit', presubmit=True), + PythonArtifact('windows', 'x64', 'Python36', presubmit=True), PythonArtifact('windows', 'x64', 'Python37'), PythonArtifact('windows', 'x64', 'Python38'), PythonArtifact('windows', 'x64', 'Python39'), - PythonArtifact('windows', 'x64', 'Python310'), - RubyArtifact('linux', 'x64'), - RubyArtifact('macos', 'x64'), - PHPArtifact('linux', 'x64'), - PHPArtifact('macos', 'x64') + PythonArtifact('windows', 'x64', 'Python310', presubmit=True), + RubyArtifact('linux', 'x64', presubmit=True), + RubyArtifact('macos', 'x64', presubmit=True), + PHPArtifact('linux', 'x64', presubmit=True), + PHPArtifact('macos', 'x64', presubmit=True) ] diff --git a/tools/run_tests/artifacts/distribtest_targets.py b/tools/run_tests/artifacts/distribtest_targets.py index ab3fc1fe5f6..fed41dd77c2 100644 --- a/tools/run_tests/artifacts/distribtest_targets.py +++ b/tools/run_tests/artifacts/distribtest_targets.py @@ -85,12 +85,15 @@ class CSharpDistribTest(object): platform, arch, docker_suffix=None, - use_dotnet_cli=False): + use_dotnet_cli=False, + presubmit=False): self.name = 'csharp_%s_%s' % (platform, arch) self.platform = platform self.arch = arch self.docker_suffix = docker_suffix self.labels = ['distribtest', 'csharp', platform, arch] + if presubmit: + self.labels.append('presubmit') self.script_suffix = '' if docker_suffix: self.name += '_%s' % docker_suffix @@ -146,7 +149,12 @@ class CSharpDistribTest(object): class PythonDistribTest(object): """Tests Python package""" - def __init__(self, platform, arch, docker_suffix, source=False): + def __init__(self, + platform, + arch, + docker_suffix, + source=False, + presubmit=False): self.source = source if source: self.name = 'python_dev_%s_%s_%s' % (platform, arch, docker_suffix) @@ -156,6 +164,8 @@ class PythonDistribTest(object): self.arch = arch self.docker_suffix = docker_suffix self.labels = ['distribtest', 'python', platform, arch, docker_suffix] + if presubmit: + self.labels.append('presubmit') def pre_build_jobspecs(self): return [] @@ -191,7 +201,8 @@ class RubyDistribTest(object): arch, docker_suffix, ruby_version=None, - source=False): + source=False, + presubmit=False): self.package_type = 'binary' if source: self.package_type = 'source' @@ -203,6 +214,8 @@ class RubyDistribTest(object): self.docker_suffix = docker_suffix self.ruby_version = ruby_version self.labels = ['distribtest', 'ruby', platform, arch, docker_suffix] + if presubmit: + self.labels.append('presubmit') def pre_build_jobspecs(self): return [] @@ -233,12 +246,14 @@ class RubyDistribTest(object): class PHP7DistribTest(object): """Tests PHP7 package""" - def __init__(self, platform, arch, docker_suffix=None): + def __init__(self, platform, arch, docker_suffix=None, presubmit=False): self.name = 'php7_%s_%s_%s' % (platform, arch, docker_suffix) self.platform = platform self.arch = arch self.docker_suffix = docker_suffix self.labels = ['distribtest', 'php', 'php7', platform, arch] + if presubmit: + self.labels.append('presubmit') if docker_suffix: self.labels.append(docker_suffix) @@ -269,7 +284,12 @@ class PHP7DistribTest(object): class CppDistribTest(object): """Tests Cpp make install by building examples.""" - def __init__(self, platform, arch, docker_suffix=None, testcase=None): + def __init__(self, + platform, + arch, + docker_suffix=None, + testcase=None, + presubmit=False): if platform == 'linux': self.name = 'cpp_%s_%s_%s_%s' % (platform, arch, docker_suffix, testcase) @@ -286,6 +306,8 @@ class CppDistribTest(object): arch, testcase, ] + if presubmit: + self.labels.append('presubmit') if docker_suffix: self.labels.append(docker_suffix) @@ -318,64 +340,125 @@ def targets(): """Gets list of supported targets""" return [ # C++ - CppDistribTest('linux', 'x64', 'jessie', 'cmake_as_submodule'), - CppDistribTest('linux', 'x64', 'stretch', 'cmake'), - CppDistribTest('linux', 'x64', 'stretch', 'cmake_as_externalproject'), - CppDistribTest('linux', 'x64', 'stretch', 'cmake_fetchcontent'), - CppDistribTest('linux', 'x64', 'stretch', 'cmake_module_install'), - CppDistribTest('linux', 'x64', 'stretch', - 'cmake_module_install_pkgconfig'), - CppDistribTest('linux', 'x64', 'stretch', 'cmake_pkgconfig'), - CppDistribTest('linux', 'x64', 'stretch_aarch64_cross', - 'cmake_aarch64_cross'), - CppDistribTest('windows', 'x86', testcase='cmake'), - CppDistribTest('windows', 'x86', testcase='cmake_as_externalproject'), + CppDistribTest('linux', + 'x64', + 'jessie', + 'cmake_as_submodule', + presubmit=True), + CppDistribTest('linux', 'x64', 'stretch', 'cmake', presubmit=True), + CppDistribTest('linux', + 'x64', + 'stretch', + 'cmake_as_externalproject', + presubmit=True), + CppDistribTest('linux', + 'x64', + 'stretch', + 'cmake_fetchcontent', + presubmit=True), + CppDistribTest('linux', + 'x64', + 'stretch', + 'cmake_module_install', + presubmit=True), + CppDistribTest('linux', + 'x64', + 'stretch', + 'cmake_module_install_pkgconfig', + presubmit=True), + CppDistribTest('linux', + 'x64', + 'stretch', + 'cmake_pkgconfig', + presubmit=True), + CppDistribTest('linux', + 'x64', + 'stretch_aarch64_cross', + 'cmake_aarch64_cross', + presubmit=True), + CppDistribTest('windows', 'x86', testcase='cmake', presubmit=True), + CppDistribTest('windows', + 'x86', + testcase='cmake_as_externalproject', + presubmit=True), # C# - CSharpDistribTest('linux', 'x64', 'jessie'), + CSharpDistribTest('linux', 'x64', 'jessie', presubmit=True), CSharpDistribTest('linux', 'x64', 'stretch'), - CSharpDistribTest('linux', 'x64', 'stretch', use_dotnet_cli=True), + CSharpDistribTest('linux', + 'x64', + 'stretch', + use_dotnet_cli=True, + presubmit=True), CSharpDistribTest('linux', 'x64', 'centos7'), CSharpDistribTest('linux', 'x64', 'ubuntu1604'), CSharpDistribTest('linux', 'x64', 'ubuntu1604', use_dotnet_cli=True), - CSharpDistribTest('linux', 'x64', 'alpine', use_dotnet_cli=True), - CSharpDistribTest('linux', 'x64', 'dotnet31', use_dotnet_cli=True), - CSharpDistribTest('linux', 'x64', 'dotnet5', use_dotnet_cli=True), - CSharpDistribTest('macos', 'x64'), - CSharpDistribTest('windows', 'x86'), - CSharpDistribTest('windows', 'x64'), + CSharpDistribTest('linux', + 'x64', + 'alpine', + use_dotnet_cli=True, + presubmit=True), + CSharpDistribTest('linux', + 'x64', + 'dotnet31', + use_dotnet_cli=True, + presubmit=True), + CSharpDistribTest('linux', + 'x64', + 'dotnet5', + use_dotnet_cli=True, + presubmit=True), + CSharpDistribTest('macos', 'x64', presubmit=True), + CSharpDistribTest('windows', 'x86', presubmit=True), + CSharpDistribTest('windows', 'x64', presubmit=True), # Python - PythonDistribTest('linux', 'x64', 'buster'), - PythonDistribTest('linux', 'x86', 'buster'), + PythonDistribTest('linux', 'x64', 'buster', presubmit=True), + PythonDistribTest('linux', 'x86', 'buster', presubmit=True), PythonDistribTest('linux', 'x64', 'centos7'), PythonDistribTest('linux', 'x64', 'fedora34'), PythonDistribTest('linux', 'x64', 'opensuse'), PythonDistribTest('linux', 'x64', 'arch'), PythonDistribTest('linux', 'x64', 'ubuntu1804'), - PythonDistribTest('linux', 'aarch64', 'python38_buster'), - PythonDistribTest('linux', 'x64', 'alpine3.7', source=True), - PythonDistribTest('linux', 'x64', 'buster', source=True), - PythonDistribTest('linux', 'x86', 'buster', source=True), + PythonDistribTest('linux', 'aarch64', 'python38_buster', + presubmit=True), + PythonDistribTest('linux', + 'x64', + 'alpine3.7', + source=True, + presubmit=True), + PythonDistribTest('linux', 'x64', 'buster', source=True, + presubmit=True), + PythonDistribTest('linux', 'x86', 'buster', source=True, + presubmit=True), PythonDistribTest('linux', 'x64', 'centos7', source=True), PythonDistribTest('linux', 'x64', 'fedora34', source=True), PythonDistribTest('linux', 'x64', 'arch', source=True), PythonDistribTest('linux', 'x64', 'ubuntu1804', source=True), # Ruby - RubyDistribTest('linux', 'x64', 'jessie', ruby_version='ruby_2_4'), + RubyDistribTest('linux', + 'x64', + 'jessie', + ruby_version='ruby_2_4', + presubmit=True), RubyDistribTest('linux', 'x64', 'jessie', ruby_version='ruby_2_5'), RubyDistribTest('linux', 'x64', 'jessie', ruby_version='ruby_2_6'), - RubyDistribTest('linux', 'x64', 'jessie', ruby_version='ruby_2_7'), + RubyDistribTest('linux', + 'x64', + 'jessie', + ruby_version='ruby_2_7', + presubmit=True), # TODO(apolcyn): add a ruby 3.0 test once protobuf adds support RubyDistribTest('linux', 'x64', 'jessie', ruby_version='ruby_2_4', - source=True), + source=True, + presubmit=True), RubyDistribTest('linux', 'x64', 'centos7'), RubyDistribTest('linux', 'x64', 'fedora23'), RubyDistribTest('linux', 'x64', 'opensuse'), RubyDistribTest('linux', 'x64', 'ubuntu1604'), - RubyDistribTest('linux', 'x64', 'ubuntu1804'), + RubyDistribTest('linux', 'x64', 'ubuntu1804', presubmit=True), # PHP7 - PHP7DistribTest('linux', 'x64', 'stretch'), - PHP7DistribTest('macos', 'x64'), + PHP7DistribTest('linux', 'x64', 'stretch', presubmit=True), + PHP7DistribTest('macos', 'x64', presubmit=True), ]