The C based gRPC (C++, Python, Ruby, Objective-C, PHP, C#) https://grpc.io/
 
 
 
 
 
 

480 lines
18 KiB

#!/usr/bin/env python3
# Copyright 2016 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Definition of targets to build artifacts."""
import os.path
import random
import string
import sys
sys.path.insert(0, os.path.abspath(".."))
import python_utils.jobset as jobset
_LATEST_MANYLINUX = "manylinux2014"
def create_docker_jobspec(
name,
dockerfile_dir,
shell_command,
environ={},
flake_retries=0,
timeout_retries=0,
timeout_seconds=30 * 60,
extra_docker_args=None,
verbose_success=False,
):
"""Creates jobspec for a task running under docker."""
environ = environ.copy()
environ["ARTIFACTS_OUT"] = "artifacts/%s" % name
docker_args = []
for k, v in list(environ.items()):
docker_args += ["-e", "%s=%s" % (k, v)]
docker_env = {
"DOCKERFILE_DIR": dockerfile_dir,
"DOCKER_RUN_SCRIPT": "tools/run_tests/dockerize/docker_run.sh",
"DOCKER_RUN_SCRIPT_COMMAND": shell_command,
"OUTPUT_DIR": "artifacts",
}
if extra_docker_args is not None:
docker_env["EXTRA_DOCKER_ARGS"] = extra_docker_args
jobspec = jobset.JobSpec(
cmdline=["tools/run_tests/dockerize/build_and_run_docker.sh"]
+ docker_args,
environ=docker_env,
shortname="build_artifact.%s" % (name),
timeout_seconds=timeout_seconds,
flake_retries=flake_retries,
timeout_retries=timeout_retries,
verbose_success=verbose_success,
)
return jobspec
def create_jobspec(
name,
cmdline,
environ={},
shell=False,
flake_retries=0,
timeout_retries=0,
timeout_seconds=30 * 60,
use_workspace=False,
cpu_cost=1.0,
verbose_success=False,
):
"""Creates jobspec."""
environ = environ.copy()
if use_workspace:
environ["WORKSPACE_NAME"] = "workspace_%s" % name
environ["ARTIFACTS_OUT"] = os.path.join("..", "artifacts", name)
cmdline = [
"bash",
"tools/run_tests/artifacts/run_in_workspace.sh",
] + cmdline
else:
environ["ARTIFACTS_OUT"] = os.path.join("artifacts", name)
jobspec = jobset.JobSpec(
cmdline=cmdline,
environ=environ,
shortname="build_artifact.%s" % (name),
timeout_seconds=timeout_seconds,
flake_retries=flake_retries,
timeout_retries=timeout_retries,
shell=shell,
cpu_cost=cpu_cost,
verbose_success=verbose_success,
)
return jobspec
_MACOS_COMPAT_FLAG = "-mmacosx-version-min=10.10"
_ARCH_FLAG_MAP = {"x86": "-m32", "x64": "-m64"}
class PythonArtifact:
"""Builds Python artifacts."""
def __init__(self, platform, arch, py_version, presubmit=False):
self.name = "python_%s_%s_%s" % (platform, arch, py_version)
self.platform = platform
self.arch = arch
self.labels = ["artifact", "python", platform, arch, py_version]
if presubmit:
self.labels.append("presubmit")
self.py_version = py_version
if platform == _LATEST_MANYLINUX:
self.labels.append("latest-manylinux")
if "manylinux" in platform:
self.labels.append("linux")
if "linux_extra" in platform:
# linux_extra wheels used to be built by a separate kokoro job.
# Their build is now much faster, so they can be included
# in the regular artifact build.
self.labels.append("linux")
if "musllinux" in platform:
self.labels.append("linux")
def pre_build_jobspecs(self):
return []
def build_jobspec(self, inner_jobs=None):
environ = {}
if inner_jobs is not None:
# set number of parallel jobs when building native extension
# building the native extension is the most time-consuming part of the build
environ["GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS"] = str(inner_jobs)
if self.platform == "macos":
environ["ARCHFLAGS"] = "-arch arm64 -arch x86_64"
environ["GRPC_UNIVERSAL2_REPAIR"] = "true"
environ["GRPC_BUILD_WITH_BORING_SSL_ASM"] = "false"
if self.platform == "linux_extra":
# Crosscompilation build for armv7 (e.g. Raspberry Pi)
environ["PYTHON"] = "/opt/python/{}/bin/python3".format(
self.py_version
)
environ["PIP"] = "/opt/python/{}/bin/pip3".format(self.py_version)
environ["GRPC_SKIP_PIP_CYTHON_UPGRADE"] = "TRUE"
environ["GRPC_SKIP_TWINE_CHECK"] = "TRUE"
return create_docker_jobspec(
self.name,
"tools/dockerfile/grpc_artifact_python_linux_{}".format(
self.arch
),
"tools/run_tests/artifacts/build_artifact_python.sh",
environ=environ,
timeout_seconds=60 * 60,
)
elif "manylinux" in self.platform:
if self.arch == "x86":
environ["SETARCH_CMD"] = "linux32"
# Inside the manylinux container, the python installations are located in
# special places...
environ["PYTHON"] = "/opt/python/{}/bin/python".format(
self.py_version
)
environ["PIP"] = "/opt/python/{}/bin/pip".format(self.py_version)
environ["GRPC_SKIP_PIP_CYTHON_UPGRADE"] = "TRUE"
if self.arch == "aarch64":
environ["GRPC_SKIP_TWINE_CHECK"] = "TRUE"
# As we won't strip the binary with auditwheel (see below), strip
# it at link time.
environ["LDFLAGS"] = "-s"
else:
# only run auditwheel if we're not crosscompiling
environ["GRPC_RUN_AUDITWHEEL_REPAIR"] = "TRUE"
# only build the packages that depend on grpcio-tools
# if we're not crosscompiling.
# - they require protoc to run on current architecture
# - they only have sdist packages anyway, so it's useless to build them again
environ["GRPC_BUILD_GRPCIO_TOOLS_DEPENDENTS"] = "TRUE"
return create_docker_jobspec(
self.name,
"tools/dockerfile/grpc_artifact_python_%s_%s"
% (self.platform, self.arch),
"tools/run_tests/artifacts/build_artifact_python.sh",
environ=environ,
timeout_seconds=60 * 60 * 2,
)
elif "musllinux" in self.platform:
environ["PYTHON"] = "/opt/python/{}/bin/python".format(
self.py_version
)
environ["PIP"] = "/opt/python/{}/bin/pip".format(self.py_version)
environ["GRPC_SKIP_PIP_CYTHON_UPGRADE"] = "TRUE"
environ["GRPC_RUN_AUDITWHEEL_REPAIR"] = "TRUE"
environ["GRPC_PYTHON_BUILD_WITH_STATIC_LIBSTDCXX"] = "TRUE"
return create_docker_jobspec(
self.name,
"tools/dockerfile/grpc_artifact_python_%s_%s"
% (self.platform, self.arch),
"tools/run_tests/artifacts/build_artifact_python.sh",
environ=environ,
timeout_seconds=60 * 60 * 2,
)
elif self.platform == "windows":
environ["EXT_COMPILER"] = "msvc"
# For some reason, the batch script %random% always runs with the same
# seed. We create a random temp-dir here
dir = "".join(
random.choice(string.ascii_uppercase) for _ in range(10)
)
return create_jobspec(
self.name,
[
"tools\\run_tests\\artifacts\\build_artifact_python.bat",
self.py_version,
"32" if self.arch == "x86" else "64",
],
environ=environ,
timeout_seconds=45 * 60,
use_workspace=True,
)
else:
environ["PYTHON"] = self.py_version
environ["SKIP_PIP_INSTALL"] = "TRUE"
return create_jobspec(
self.name,
["tools/run_tests/artifacts/build_artifact_python.sh"],
environ=environ,
timeout_seconds=60 * 60 * 2,
use_workspace=True,
)
def __str__(self):
return self.name
class RubyArtifact:
"""Builds ruby native gem."""
def __init__(self, platform, gem_platform, presubmit=False):
self.name = "ruby_native_gem_%s_%s" % (platform, gem_platform)
self.platform = platform
self.gem_platform = gem_platform
self.labels = ["artifact", "ruby", platform, gem_platform]
if presubmit:
self.labels.append("presubmit")
def pre_build_jobspecs(self):
return []
def build_jobspec(self, inner_jobs=None):
environ = {}
if inner_jobs is not None:
# set number of parallel jobs when building native extension
environ["GRPC_RUBY_BUILD_PROCS"] = str(inner_jobs)
# Ruby build uses docker internally and docker cannot be nested.
# We are using a custom workspace instead.
return create_jobspec(
self.name,
[
"tools/run_tests/artifacts/build_artifact_ruby.sh",
self.gem_platform,
],
use_workspace=True,
timeout_seconds=90 * 60,
environ=environ,
)
class PHPArtifact:
"""Builds PHP PECL package"""
def __init__(self, platform, arch, presubmit=False):
self.name = "php_pecl_package_{0}_{1}".format(platform, arch)
self.platform = platform
self.arch = arch
self.labels = ["artifact", "php", platform, arch]
if presubmit:
self.labels.append("presubmit")
def pre_build_jobspecs(self):
return []
def build_jobspec(self, inner_jobs=None):
del inner_jobs # arg unused as PHP artifact build is basically just packing an archive
if self.platform == "linux":
return create_docker_jobspec(
self.name,
"tools/dockerfile/test/php73_zts_debian11_{}".format(self.arch),
"tools/run_tests/artifacts/build_artifact_php.sh",
)
else:
return create_jobspec(
self.name,
["tools/run_tests/artifacts/build_artifact_php.sh"],
use_workspace=True,
)
class ProtocArtifact:
"""Builds protoc and protoc-plugin artifacts"""
def __init__(self, platform, arch, presubmit=False):
self.name = "protoc_%s_%s" % (platform, arch)
self.platform = platform
self.arch = arch
self.labels = ["artifact", "protoc", platform, arch]
if presubmit:
self.labels.append("presubmit")
def pre_build_jobspecs(self):
return []
def build_jobspec(self, inner_jobs=None):
environ = {}
if inner_jobs is not None:
# set number of parallel jobs when building protoc
environ["GRPC_PROTOC_BUILD_COMPILER_JOBS"] = str(inner_jobs)
if self.platform != "windows":
environ["CXXFLAGS"] = ""
environ["LDFLAGS"] = ""
if self.platform == "linux":
dockerfile_dir = (
"tools/dockerfile/grpc_artifact_centos6_{}".format(
self.arch
)
)
if self.arch == "aarch64":
# for aarch64, use a dockcross manylinux image that will
# give us both ready to use crosscompiler and sufficient backward compatibility
dockerfile_dir = (
"tools/dockerfile/grpc_artifact_protoc_aarch64"
)
environ["LDFLAGS"] += " -static-libgcc -static-libstdc++ -s"
return create_docker_jobspec(
self.name,
dockerfile_dir,
"tools/run_tests/artifacts/build_artifact_protoc.sh",
environ=environ,
)
else:
environ["CXXFLAGS"] += (
" -std=c++14 -stdlib=libc++ %s" % _MACOS_COMPAT_FLAG
)
return create_jobspec(
self.name,
["tools/run_tests/artifacts/build_artifact_protoc.sh"],
environ=environ,
timeout_seconds=60 * 60,
use_workspace=True,
)
else:
vs_tools_architecture = (
self.arch
) # architecture selector passed to vcvarsall.bat
environ["ARCHITECTURE"] = vs_tools_architecture
return create_jobspec(
self.name,
["tools\\run_tests\\artifacts\\build_artifact_protoc.bat"],
environ=environ,
use_workspace=True,
)
def __str__(self):
return self.name
def _reorder_targets_for_build_speed(targets):
"""Reorder targets to achieve optimal build speed"""
# ruby artifact build builds multiple artifacts at once, so make sure
# we start building ruby artifacts first, so that they don't end up
# being a long tail once everything else finishes.
return list(
sorted(
targets,
key=lambda target: 0 if target.name.startswith("ruby_") else 1,
)
)
def targets():
"""Gets list of supported targets"""
return _reorder_targets_for_build_speed(
[
ProtocArtifact("linux", "x64", presubmit=True),
ProtocArtifact("linux", "x86", presubmit=True),
ProtocArtifact("linux", "aarch64", presubmit=True),
ProtocArtifact("macos", "x64", presubmit=True),
ProtocArtifact("windows", "x64", presubmit=True),
ProtocArtifact("windows", "x86", presubmit=True),
PythonArtifact(
"manylinux2014", "x64", "cp37-cp37m", presubmit=True
),
PythonArtifact("manylinux2014", "x64", "cp38-cp38", presubmit=True),
PythonArtifact("manylinux2014", "x64", "cp39-cp39"),
PythonArtifact("manylinux2014", "x64", "cp310-cp310"),
PythonArtifact(
"manylinux2014", "x64", "cp311-cp311", presubmit=True
),
PythonArtifact(
"manylinux2014", "x86", "cp37-cp37m", presubmit=True
),
PythonArtifact("manylinux2014", "x86", "cp38-cp38", presubmit=True),
PythonArtifact("manylinux2014", "x86", "cp39-cp39"),
PythonArtifact("manylinux2014", "x86", "cp310-cp310"),
PythonArtifact(
"manylinux2014", "x86", "cp311-cp311", presubmit=True
),
PythonArtifact(
"manylinux2014", "aarch64", "cp37-cp37m", presubmit=True
),
PythonArtifact(
"manylinux2014", "aarch64", "cp38-cp38", presubmit=True
),
PythonArtifact("manylinux2014", "aarch64", "cp39-cp39"),
PythonArtifact("manylinux2014", "aarch64", "cp310-cp310"),
PythonArtifact("manylinux2014", "aarch64", "cp311-cp311"),
PythonArtifact(
"linux_extra", "armv7", "cp37-cp37m", presubmit=True
),
PythonArtifact("linux_extra", "armv7", "cp38-cp38"),
PythonArtifact("linux_extra", "armv7", "cp39-cp39"),
PythonArtifact("linux_extra", "armv7", "cp310-cp310"),
PythonArtifact(
"linux_extra", "armv7", "cp311-cp311", presubmit=True
),
PythonArtifact("musllinux_1_1", "x64", "cp310-cp310"),
PythonArtifact(
"musllinux_1_1", "x64", "cp311-cp311", presubmit=True
),
PythonArtifact(
"musllinux_1_1", "x64", "cp37-cp37m", presubmit=True
),
PythonArtifact("musllinux_1_1", "x64", "cp38-cp38"),
PythonArtifact("musllinux_1_1", "x64", "cp39-cp39"),
PythonArtifact("musllinux_1_1", "x86", "cp310-cp310"),
PythonArtifact(
"musllinux_1_1", "x86", "cp311-cp311", presubmit=True
),
PythonArtifact(
"musllinux_1_1", "x86", "cp37-cp37m", presubmit=True
),
PythonArtifact("musllinux_1_1", "x86", "cp38-cp38"),
PythonArtifact("musllinux_1_1", "x86", "cp39-cp39"),
PythonArtifact("macos", "x64", "python3.7", presubmit=True),
PythonArtifact("macos", "x64", "python3.8"),
PythonArtifact("macos", "x64", "python3.9"),
PythonArtifact("macos", "x64", "python3.10", presubmit=True),
PythonArtifact("macos", "x64", "python3.11", presubmit=True),
PythonArtifact("windows", "x86", "Python37_32bit", presubmit=True),
PythonArtifact("windows", "x86", "Python38_32bit"),
PythonArtifact("windows", "x86", "Python39_32bit"),
PythonArtifact("windows", "x86", "Python310_32bit"),
PythonArtifact("windows", "x86", "Python311_32bit", presubmit=True),
PythonArtifact("windows", "x64", "Python37", presubmit=True),
PythonArtifact("windows", "x64", "Python38"),
PythonArtifact("windows", "x64", "Python39"),
PythonArtifact("windows", "x64", "Python310"),
PythonArtifact("windows", "x64", "Python311", presubmit=True),
RubyArtifact("linux", "x86-mingw32", presubmit=True),
RubyArtifact("linux", "x64-mingw32", presubmit=True),
RubyArtifact("linux", "x64-mingw-ucrt", presubmit=True),
RubyArtifact("linux", "x86_64-linux", presubmit=True),
RubyArtifact("linux", "x86-linux", presubmit=True),
RubyArtifact("linux", "aarch64-linux", presubmit=True),
RubyArtifact("linux", "x86_64-darwin", presubmit=True),
RubyArtifact("linux", "arm64-darwin", presubmit=True),
PHPArtifact("linux", "x64", presubmit=True),
PHPArtifact("macos", "x64", presubmit=True),
]
)