mirror of https://github.com/grpc/grpc.git
The C based gRPC (C++, Python, Ruby, Objective-C, PHP, C#)
https://grpc.io/
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
474 lines
15 KiB
474 lines
15 KiB
#!/usr/bin/env python3 |
|
# Copyright 2016 gRPC authors. |
|
# |
|
# Licensed under the Apache License, Version 2.0 (the "License"); |
|
# you may not use this file except in compliance with the License. |
|
# You may obtain a copy of the License at |
|
# |
|
# http://www.apache.org/licenses/LICENSE-2.0 |
|
# |
|
# Unless required by applicable law or agreed to in writing, software |
|
# distributed under the License is distributed on an "AS IS" BASIS, |
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
# See the License for the specific language governing permissions and |
|
# limitations under the License. |
|
"""Definition of targets run distribution package tests.""" |
|
|
|
import os.path |
|
import sys |
|
|
|
sys.path.insert(0, os.path.abspath("..")) |
|
import python_utils.jobset as jobset |
|
|
|
|
|
def create_docker_jobspec( |
|
name, |
|
dockerfile_dir, |
|
shell_command, |
|
environ={}, |
|
flake_retries=0, |
|
timeout_retries=0, |
|
copy_rel_path=None, |
|
timeout_seconds=30 * 60, |
|
): |
|
"""Creates jobspec for a task running under docker.""" |
|
environ = environ.copy() |
|
# the entire repo will be cloned if copy_rel_path is not set. |
|
if copy_rel_path: |
|
environ["RELATIVE_COPY_PATH"] = copy_rel_path |
|
|
|
docker_args = [] |
|
for k, v in list(environ.items()): |
|
docker_args += ["-e", "%s=%s" % (k, v)] |
|
docker_env = { |
|
"DOCKERFILE_DIR": dockerfile_dir, |
|
"DOCKER_RUN_SCRIPT": "tools/run_tests/dockerize/docker_run.sh", |
|
"DOCKER_RUN_SCRIPT_COMMAND": shell_command, |
|
} |
|
jobspec = jobset.JobSpec( |
|
cmdline=["tools/run_tests/dockerize/build_and_run_docker.sh"] |
|
+ docker_args, |
|
environ=docker_env, |
|
shortname="distribtest.%s" % (name), |
|
timeout_seconds=timeout_seconds, |
|
flake_retries=flake_retries, |
|
timeout_retries=timeout_retries, |
|
) |
|
return jobspec |
|
|
|
|
|
def create_jobspec( |
|
name, |
|
cmdline, |
|
environ=None, |
|
shell=False, |
|
flake_retries=0, |
|
timeout_retries=0, |
|
use_workspace=False, |
|
timeout_seconds=10 * 60, |
|
): |
|
"""Creates jobspec.""" |
|
environ = environ.copy() |
|
if use_workspace: |
|
environ["WORKSPACE_NAME"] = "workspace_%s" % name |
|
cmdline = [ |
|
"bash", |
|
"tools/run_tests/artifacts/run_in_workspace.sh", |
|
] + cmdline |
|
jobspec = jobset.JobSpec( |
|
cmdline=cmdline, |
|
environ=environ, |
|
shortname="distribtest.%s" % (name), |
|
timeout_seconds=timeout_seconds, |
|
flake_retries=flake_retries, |
|
timeout_retries=timeout_retries, |
|
shell=shell, |
|
) |
|
return jobspec |
|
|
|
|
|
class CSharpDistribTest(object): |
|
"""Tests C# NuGet package""" |
|
|
|
def __init__( |
|
self, |
|
platform, |
|
arch, |
|
docker_suffix=None, |
|
use_dotnet_cli=False, |
|
presubmit=False, |
|
): |
|
self.name = "csharp_%s_%s" % (platform, arch) |
|
self.platform = platform |
|
self.arch = arch |
|
self.docker_suffix = docker_suffix |
|
self.labels = ["distribtest", "csharp", platform, arch] |
|
if presubmit: |
|
self.labels.append("presubmit") |
|
self.script_suffix = "" |
|
if docker_suffix: |
|
self.name += "_%s" % docker_suffix |
|
self.labels.append(docker_suffix) |
|
if use_dotnet_cli: |
|
self.name += "_dotnetcli" |
|
self.script_suffix = "_dotnetcli" |
|
self.labels.append("dotnetcli") |
|
else: |
|
self.labels.append("olddotnet") |
|
|
|
def pre_build_jobspecs(self): |
|
return [] |
|
|
|
def build_jobspec(self, inner_jobs=None): |
|
del inner_jobs # arg unused as there is little opportunity for parallelizing whats inside the distribtests |
|
if self.platform == "linux": |
|
return create_docker_jobspec( |
|
self.name, |
|
"tools/dockerfile/distribtest/csharp_%s_%s" |
|
% (self.docker_suffix, self.arch), |
|
"test/distrib/csharp/run_distrib_test%s.sh" |
|
% self.script_suffix, |
|
copy_rel_path="test/distrib", |
|
) |
|
elif self.platform == "macos": |
|
return create_jobspec( |
|
self.name, |
|
[ |
|
"test/distrib/csharp/run_distrib_test%s.sh" |
|
% self.script_suffix |
|
], |
|
environ={ |
|
"EXTERNAL_GIT_ROOT": "../../../..", |
|
"SKIP_NETCOREAPP21_DISTRIBTEST": "1", |
|
"SKIP_NET50_DISTRIBTEST": "1", |
|
}, |
|
use_workspace=True, |
|
) |
|
elif self.platform == "windows": |
|
# TODO(jtattermusch): re-enable windows distribtest |
|
return create_jobspec( |
|
self.name, |
|
["bash", "tools/run_tests/artifacts/run_distribtest_csharp.sh"], |
|
environ={}, |
|
use_workspace=True, |
|
) |
|
else: |
|
raise Exception("Not supported yet.") |
|
|
|
def __str__(self): |
|
return self.name |
|
|
|
|
|
class PythonDistribTest(object): |
|
"""Tests Python package""" |
|
|
|
def __init__( |
|
self, platform, arch, docker_suffix, source=False, presubmit=False |
|
): |
|
self.source = source |
|
if source: |
|
self.name = "python_dev_%s_%s_%s" % (platform, arch, docker_suffix) |
|
else: |
|
self.name = "python_%s_%s_%s" % (platform, arch, docker_suffix) |
|
self.platform = platform |
|
self.arch = arch |
|
self.docker_suffix = docker_suffix |
|
self.labels = ["distribtest", "python", platform, arch, docker_suffix] |
|
if presubmit: |
|
self.labels.append("presubmit") |
|
|
|
def pre_build_jobspecs(self): |
|
return [] |
|
|
|
def build_jobspec(self, inner_jobs=None): |
|
# TODO(jtattermusch): honor inner_jobs arg for this task. |
|
del inner_jobs |
|
if not self.platform == "linux": |
|
raise Exception("Not supported yet.") |
|
|
|
if self.source: |
|
return create_docker_jobspec( |
|
self.name, |
|
"tools/dockerfile/distribtest/python_dev_%s_%s" |
|
% (self.docker_suffix, self.arch), |
|
"test/distrib/python/run_source_distrib_test.sh", |
|
copy_rel_path="test/distrib", |
|
) |
|
else: |
|
return create_docker_jobspec( |
|
self.name, |
|
"tools/dockerfile/distribtest/python_%s_%s" |
|
% (self.docker_suffix, self.arch), |
|
"test/distrib/python/run_binary_distrib_test.sh", |
|
copy_rel_path="test/distrib", |
|
) |
|
|
|
def __str__(self): |
|
return self.name |
|
|
|
|
|
class RubyDistribTest(object): |
|
"""Tests Ruby package""" |
|
|
|
def __init__( |
|
self, |
|
platform, |
|
arch, |
|
docker_suffix, |
|
ruby_version=None, |
|
source=False, |
|
presubmit=False, |
|
): |
|
self.package_type = "binary" |
|
if source: |
|
self.package_type = "source" |
|
self.name = "ruby_%s_%s_%s_version_%s_package_type_%s" % ( |
|
platform, |
|
arch, |
|
docker_suffix, |
|
ruby_version or "unspecified", |
|
self.package_type, |
|
) |
|
self.platform = platform |
|
self.arch = arch |
|
self.docker_suffix = docker_suffix |
|
self.ruby_version = ruby_version |
|
self.labels = ["distribtest", "ruby", platform, arch, docker_suffix] |
|
if presubmit: |
|
self.labels.append("presubmit") |
|
|
|
def pre_build_jobspecs(self): |
|
return [] |
|
|
|
def build_jobspec(self, inner_jobs=None): |
|
# TODO(jtattermusch): honor inner_jobs arg for this task. |
|
del inner_jobs |
|
arch_to_gem_arch = { |
|
"x64": "x86_64", |
|
"x86": "x86", |
|
} |
|
if not self.platform == "linux": |
|
raise Exception("Not supported yet.") |
|
|
|
dockerfile_name = "tools/dockerfile/distribtest/ruby_%s_%s" % ( |
|
self.docker_suffix, |
|
self.arch, |
|
) |
|
if self.ruby_version is not None: |
|
dockerfile_name += "_%s" % self.ruby_version |
|
return create_docker_jobspec( |
|
self.name, |
|
dockerfile_name, |
|
"test/distrib/ruby/run_distrib_test.sh %s %s %s" |
|
% (arch_to_gem_arch[self.arch], self.platform, self.package_type), |
|
copy_rel_path="test/distrib", |
|
) |
|
|
|
def __str__(self): |
|
return self.name |
|
|
|
|
|
class PHP7DistribTest(object): |
|
"""Tests PHP7 package""" |
|
|
|
def __init__(self, platform, arch, docker_suffix=None, presubmit=False): |
|
self.name = "php7_%s_%s_%s" % (platform, arch, docker_suffix) |
|
self.platform = platform |
|
self.arch = arch |
|
self.docker_suffix = docker_suffix |
|
self.labels = ["distribtest", "php", "php7", platform, arch] |
|
if presubmit: |
|
self.labels.append("presubmit") |
|
if docker_suffix: |
|
self.labels.append(docker_suffix) |
|
|
|
def pre_build_jobspecs(self): |
|
return [] |
|
|
|
def build_jobspec(self, inner_jobs=None): |
|
# TODO(jtattermusch): honor inner_jobs arg for this task. |
|
del inner_jobs |
|
if self.platform == "linux": |
|
return create_docker_jobspec( |
|
self.name, |
|
"tools/dockerfile/distribtest/php7_%s_%s" |
|
% (self.docker_suffix, self.arch), |
|
"test/distrib/php/run_distrib_test.sh", |
|
copy_rel_path="test/distrib", |
|
) |
|
elif self.platform == "macos": |
|
return create_jobspec( |
|
self.name, |
|
["test/distrib/php/run_distrib_test_macos.sh"], |
|
environ={"EXTERNAL_GIT_ROOT": "../../../.."}, |
|
timeout_seconds=20 * 60, |
|
use_workspace=True, |
|
) |
|
else: |
|
raise Exception("Not supported yet.") |
|
|
|
def __str__(self): |
|
return self.name |
|
|
|
|
|
class CppDistribTest(object): |
|
"""Tests Cpp make install by building examples.""" |
|
|
|
def __init__( |
|
self, platform, arch, docker_suffix=None, testcase=None, presubmit=False |
|
): |
|
if platform == "linux": |
|
self.name = "cpp_%s_%s_%s_%s" % ( |
|
platform, |
|
arch, |
|
docker_suffix, |
|
testcase, |
|
) |
|
else: |
|
self.name = "cpp_%s_%s_%s" % (platform, arch, testcase) |
|
self.platform = platform |
|
self.arch = arch |
|
self.docker_suffix = docker_suffix |
|
self.testcase = testcase |
|
self.labels = [ |
|
"distribtest", |
|
"cpp", |
|
platform, |
|
arch, |
|
testcase, |
|
] |
|
if presubmit: |
|
self.labels.append("presubmit") |
|
if docker_suffix: |
|
self.labels.append(docker_suffix) |
|
|
|
def pre_build_jobspecs(self): |
|
return [] |
|
|
|
def build_jobspec(self, inner_jobs=None): |
|
environ = {} |
|
if inner_jobs is not None: |
|
# set number of parallel jobs for the C++ build |
|
environ["GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS"] = str( |
|
inner_jobs |
|
) |
|
|
|
if self.platform == "linux": |
|
return create_docker_jobspec( |
|
self.name, |
|
"tools/dockerfile/distribtest/cpp_%s_%s" |
|
% (self.docker_suffix, self.arch), |
|
"test/distrib/cpp/run_distrib_test_%s.sh" % self.testcase, |
|
timeout_seconds=45 * 60, |
|
) |
|
elif self.platform == "windows": |
|
return create_jobspec( |
|
self.name, |
|
["test\\distrib\\cpp\\run_distrib_test_%s.bat" % self.testcase], |
|
environ={}, |
|
timeout_seconds=45 * 60, |
|
use_workspace=True, |
|
) |
|
else: |
|
raise Exception("Not supported yet.") |
|
|
|
def __str__(self): |
|
return self.name |
|
|
|
|
|
def targets(): |
|
"""Gets list of supported targets""" |
|
return [ |
|
# C++ |
|
CppDistribTest("linux", "x64", "debian10", "cmake", presubmit=True), |
|
CppDistribTest( |
|
"linux", "x64", "debian10", "cmake_as_submodule", presubmit=True |
|
), |
|
CppDistribTest( |
|
"linux", |
|
"x64", |
|
"debian10", |
|
"cmake_as_externalproject", |
|
presubmit=True, |
|
), |
|
CppDistribTest( |
|
"linux", "x64", "debian10", "cmake_fetchcontent", presubmit=True |
|
), |
|
CppDistribTest( |
|
"linux", "x64", "debian10", "cmake_module_install", presubmit=True |
|
), |
|
CppDistribTest( |
|
"linux", "x64", "debian10", "cmake_pkgconfig", presubmit=True |
|
), |
|
CppDistribTest( |
|
"linux", |
|
"x64", |
|
"debian10_aarch64_cross", |
|
"cmake_aarch64_cross", |
|
presubmit=True, |
|
), |
|
CppDistribTest("windows", "x86", testcase="cmake", presubmit=True), |
|
CppDistribTest( |
|
"windows", |
|
"x86", |
|
testcase="cmake_as_externalproject", |
|
presubmit=True, |
|
), |
|
# C# |
|
CSharpDistribTest( |
|
"linux", "x64", "debian10", use_dotnet_cli=True, presubmit=True |
|
), |
|
CSharpDistribTest("linux", "x64", "ubuntu1604", use_dotnet_cli=True), |
|
CSharpDistribTest( |
|
"linux", "x64", "alpine", use_dotnet_cli=True, presubmit=True |
|
), |
|
CSharpDistribTest( |
|
"linux", "x64", "dotnet31", use_dotnet_cli=True, presubmit=True |
|
), |
|
CSharpDistribTest( |
|
"linux", "x64", "dotnet5", use_dotnet_cli=True, presubmit=True |
|
), |
|
CSharpDistribTest("macos", "x64", use_dotnet_cli=True, presubmit=True), |
|
CSharpDistribTest("windows", "x86", presubmit=True), |
|
CSharpDistribTest("windows", "x64", presubmit=True), |
|
# Python |
|
PythonDistribTest("linux", "x64", "buster", presubmit=True), |
|
PythonDistribTest("linux", "x86", "buster", presubmit=True), |
|
PythonDistribTest("linux", "x64", "fedora34"), |
|
PythonDistribTest("linux", "x64", "arch"), |
|
PythonDistribTest("linux", "x64", "alpine"), |
|
PythonDistribTest("linux", "x64", "ubuntu2004"), |
|
PythonDistribTest( |
|
"linux", "aarch64", "python38_buster", presubmit=True |
|
), |
|
PythonDistribTest( |
|
"linux", "x64", "alpine3.7", source=True, presubmit=True |
|
), |
|
PythonDistribTest( |
|
"linux", "x64", "buster", source=True, presubmit=True |
|
), |
|
PythonDistribTest( |
|
"linux", "x86", "buster", source=True, presubmit=True |
|
), |
|
PythonDistribTest("linux", "x64", "fedora34", source=True), |
|
PythonDistribTest("linux", "x64", "arch", source=True), |
|
PythonDistribTest("linux", "x64", "ubuntu2004", source=True), |
|
# Ruby |
|
RubyDistribTest( |
|
"linux", |
|
"x64", |
|
"debian10", |
|
ruby_version="ruby_2_6", |
|
source=True, |
|
presubmit=True, |
|
), |
|
RubyDistribTest( |
|
"linux", "x64", "debian10", ruby_version="ruby_2_7", presubmit=True |
|
), |
|
RubyDistribTest("linux", "x64", "centos7"), |
|
RubyDistribTest("linux", "x64", "ubuntu1604"), |
|
RubyDistribTest("linux", "x64", "ubuntu1804", presubmit=True), |
|
# PHP7 |
|
PHP7DistribTest("linux", "x64", "debian10", presubmit=True), |
|
PHP7DistribTest("macos", "x64", presubmit=True), |
|
]
|
|
|